Created by: chang-png

{'AUG': {'AUG_METHOD': 'unpadding', 'FIX_RESIZE_SIZE': (512, 512), 'FLIP': False, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': False, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'BATCH_SIZE': 2, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATA_DIM': 3, 'DATA_DIR': './dataset/hongguHRNET', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 30, 'PADDING_VALUE': [127.5, 127.5, 127.5], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './dataset/hongguHRNET/test_list.txt', 'TEST_TOTAL_IMAGES': 772, 'TRAIN_FILE_LIST': './dataset/hongguHRNET/train_list.txt', 'TRAIN_TOTAL_IMAGES': 2317, 'VAL_FILE_LIST': './dataset/hongguHRNET/val_list.txt', 'VAL_TOTAL_IMAGES': 772, 'VIS_FILE_LIST': './dataset/hongguHRNET/test_list.txt'}, 'EVAL_CROP_SIZE': (512, 512), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.5, 0.5, 0.5], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLAB': {'ASPP_WITH_SEP_CONV': True, 'BACKBONE': 'xception_65', 'DECODER_USE_SEP_CONV': True, 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [32, 64], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [32, 64, 128], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [32, 64, 128, 256], 'NUM_MODULES': 3}}, 'ICNET': {'DEPTH_MULTIPLIER': 0.5, 'LAYERS': 50}, 'MODEL_NAME': 'hrnet', 'MULTI_LOSS_WEIGHT': [1.0], 'PSPNET': {'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC', 'UNET': {'UPSAMPLE_MODE': 'bilinear'}}, 'NUM_TRAINERS': 1, 'SLIM': {'KNOWLEDGE_DISTILL': False, 'KNOWLEDGE_DISTILL_IS_TEACHER': False, 'KNOWLEDGE_DISTILL_TEACHER_MODEL_DIR': '', 'NAS_ADDRESS': '', 'NAS_IS_SERVER': True, 'NAS_PORT': 23333, 'NAS_SEARCH_STEPS': 100, 'NAS_SPACE_NAME': '', 'NAS_START_EVAL_EPOCH': 0, 'PREPROCESS': False, 'PRUNE_PARAMS': '', 'PRUNE_RATIOS': []}, 'SOLVER': {'BEGIN_EPOCH': 1, 'CROSS_ENTROPY_WEIGHT': None, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LOSS_WEIGHT': {'BCE_LOSS': 1, 'DICE_LOSS': 1, 'LOVASZ_HINGE_LOSS': 1, 'LOVASZ_SOFTMAX_LOSS': 1, 'SOFTMAX_LOSS': 1}, 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 2, 'OPTIMIZER': 'adam', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.5, 0.5, 0.5], 'TEST': {'TEST_MODEL': './saved_model/hrnet_optic/final'}, 'TRAIN': {'MODEL_SAVE_DIR': './saved_model/hrnet_optic/', 'PRETRAINED_MODEL_DIR': './pretrained_model/hrnet_w18_bn_cityscapes/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': False}, 'TRAINER_ID': 0, 'TRAIN_CROP_SIZE': (512, 512)} #Device count: 1 batch_size_per_dev: 2 W0628 19:30:40.148244 981 device_context.cc:252] Please NOTE: device: 0, CUDA Capability: 70, Driver API Version: 9.2, Runtime API Version: 9.0 W0628 19:30:40.152480 981 device_context.cc:260] device: 0, cuDNN Version: 7.3. Pretrained model dir ./pretrained_model/hrnet_w18_bn_cityscapes/ not exists, training from scratch... Use multi-thread reader epoch=1 step=10 lr=0.00100 loss=1.8163 step/sec=1.999 | ETA 00:19:13 epoch=1 step=20 lr=0.00099 loss=0.3312 step/sec=5.267 | ETA 00:07:15 epoch=1 step=30 lr=0.00099 loss=0.0831 step/sec=5.287 | ETA 00:07:12 epoch=1 step=40 lr=0.00098 loss=0.3121 step/sec=5.156 | ETA 00:07:21 epoch=1 step=50 lr=0.00098 loss=0.5117 step/sec=5.266 | ETA 00:07:10 epoch=1 step=60 lr=0.00098 loss=0.2998 step/sec=5.084 | ETA 00:07:23 epoch=1 step=70 lr=0.00097 loss=0.0360 step/sec=4.989 | ETA 00:07:30 epoch=1 step=80 lr=0.00097 loss=0.1959 step/sec=4.887 | ETA 00:07:37 epoch=1 step=90 lr=0.00097 loss=0.3038 step/sec=5.207 | ETA 00:07:07 epoch=1 step=100 lr=0.00096 loss=0.1172 step/sec=5.181 | ETA 00:07:07 epoch=1 step=110 lr=0.00096 loss=0.0242 step/sec=4.726 | ETA 00:07:46 epoch=1 step=120 lr=0.00095 loss=0.0439 step/sec=4.599 | ETA 00:07:57 epoch=1 step=130 lr=0.00095 loss=0.3059 step/sec=5.155 | ETA 00:07:04 epoch=1 step=140 lr=0.00095 loss=0.4600 step/sec=5.293 | ETA 00:06:51 epoch=1 step=150 lr=0.00094 loss=0.3119 step/sec=5.054 | ETA 00:07:08 epoch=1 step=160 lr=0.00094 loss=0.0946 step/sec=4.800 | ETA 00:07:29 epoch=1 step=170 lr=0.00093 loss=0.0491 step/sec=5.027 | ETA 00:07:06 epoch=1 step=180 lr=0.00093 loss=0.2321 step/sec=5.330 | ETA 00:06:40 epoch=1 step=190 lr=0.00093 loss=0.2925 step/sec=4.854 | ETA 00:07:17 epoch=1 step=200 lr=0.00092 loss=0.2853 step/sec=4.964 | ETA 00:07:06 epoch=1 step=210 lr=0.00092 loss=0.2620 step/sec=5.199 | ETA 00:06:45 epoch=1 step=220 lr=0.00091 loss=0.2789 step/sec=5.323 | ETA 00:06:33 epoch=1 step=230 lr=0.00091 loss=0.2395 step/sec=5.179 | ETA 00:06:42 epoch=1 step=240 lr=0.00091 loss=0.0829 step/sec=5.421 | ETA 00:06:22 epoch=1 step=250 lr=0.00090 loss=0.3178 step/sec=5.494 | ETA 00:06:16 epoch=1 step=260 lr=0.00090 loss=0.0716 step/sec=5.501 | ETA 00:06:13 epoch=1 step=270 lr=0.00089 loss=0.2129 step/sec=5.517 | ETA 00:06:10 epoch=1 step=280 lr=0.00089 loss=0.2477 step/sec=5.573 | ETA 00:06:05 epoch=1 step=290 lr=0.00089 loss=0.3050 step/sec=4.993 | ETA 00:06:45 epoch=1 step=300 lr=0.00088 loss=0.2109 step/sec=4.979 | ETA 00:06:44 epoch=1 step=310 lr=0.00088 loss=0.0884 step/sec=5.377 | ETA 00:06:13 epoch=1 step=320 lr=0.00088 loss=0.1586 step/sec=5.015 | ETA 00:06:38 epoch=1 step=330 lr=0.00087 loss=0.1708 step/sec=5.033 | ETA 00:06:34 epoch=1 step=340 lr=0.00087 loss=0.2549 step/sec=4.560 | ETA 00:07:13 epoch=1 step=350 lr=0.00086 loss=0.2124 step/sec=4.593 | ETA 00:07:08 epoch=1 step=360 lr=0.00086 loss=0.1506 step/sec=4.632 | ETA 00:07:02 epoch=1 step=370 lr=0.00086 loss=0.0837 step/sec=5.515 | ETA 00:05:52 epoch=1 step=380 lr=0.00085 loss=0.3997 step/sec=5.341 | ETA 00:06:02 epoch=1 step=390 lr=0.00085 loss=0.1531 step/sec=5.300 | ETA 00:06:03 epoch=1 step=400 lr=0.00084 loss=0.0563 step/sec=5.238 | ETA 00:06:05 epoch=1 step=410 lr=0.00084 loss=0.1846 step/sec=5.344 | ETA 00:05:56 epoch=1 step=420 lr=0.00084 loss=0.1685 step/sec=5.201 | ETA 00:06:04 epoch=1 step=430 lr=0.00083 loss=0.2323 step/sec=5.364 | ETA 00:05:51 epoch=1 step=440 lr=0.00083 loss=0.1423 step/sec=5.428 | ETA 00:05:45 epoch=1 step=450 lr=0.00082 loss=0.4865 step/sec=5.043 | ETA 00:06:10 epoch=1 step=460 lr=0.00082 loss=0.2687 step/sec=5.324 | ETA 00:05:48 epoch=1 step=470 lr=0.00082 loss=0.0987 step/sec=5.046 | ETA 00:06:05 epoch=1 step=480 lr=0.00081 loss=0.2082 step/sec=5.269 | ETA 00:05:48 epoch=1 step=490 lr=0.00081 loss=0.2386 step/sec=5.502 | ETA 00:05:31 epoch=1 step=500 lr=0.00080 loss=0.3380 step/sec=5.510 | ETA 00:05:29 epoch=1 step=510 lr=0.00080 loss=0.3068 step/sec=5.513 | ETA 00:05:27 epoch=1 step=520 lr=0.00080 loss=0.2105 step/sec=5.412 | ETA 00:05:31 epoch=1 step=530 lr=0.00079 loss=0.2915 step/sec=5.391 | ETA 00:05:31 epoch=1 step=540 lr=0.00079 loss=0.1835 step/sec=5.384 | ETA 00:05:29 epoch=1 step=550 lr=0.00078 loss=0.2699 step/sec=5.254 | ETA 00:05:36 epoch=1 step=560 lr=0.00078 loss=0.1210 step/sec=5.078 | ETA 00:05:45 epoch=1 step=570 lr=0.00078 loss=0.1623 step/sec=5.164 | ETA 00:05:38 epoch=1 step=580 lr=0.00077 loss=0.1774 step/sec=5.444 | ETA 00:05:18 epoch=1 step=590 lr=0.00077 loss=0.2081 step/sec=5.479 | ETA 00:05:15 epoch=1 step=600 lr=0.00076 loss=0.2097 step/sec=5.336 | ETA 00:05:21 epoch=1 step=610 lr=0.00076 loss=0.2646 step/sec=5.413 | ETA 00:05:15 epoch=1 step=620 lr=0.00076 loss=0.1707 step/sec=5.412 | ETA 00:05:13 epoch=1 step=630 lr=0.00075 loss=0.2040 step/sec=5.393 | ETA 00:05:12 epoch=1 step=640 lr=0.00075 loss=0.1695 step/sec=5.425 | ETA 00:05:08 epoch=1 step=650 lr=0.00074 loss=0.2297 step/sec=5.537 | ETA 00:05:00 epoch=1 step=660 lr=0.00074 loss=0.0697 step/sec=5.231 | ETA 00:05:16 epoch=1 step=670 lr=0.00074 loss=0.1947 step/sec=5.392 | ETA 00:05:05 epoch=1 step=680 lr=0.00073 loss=0.1198 step/sec=5.136 | ETA 00:05:18 epoch=1 step=690 lr=0.00073 loss=0.2839 step/sec=5.338 | ETA 00:05:04 epoch=1 step=700 lr=0.00072 loss=0.0373 step/sec=5.415 | ETA 00:04:58 epoch=1 step=710 lr=0.00072 loss=0.2493 step/sec=5.475 | ETA 00:04:53 epoch=1 step=720 lr=0.00072 loss=0.1024 step/sec=5.315 | ETA 00:05:00 epoch=1 step=730 lr=0.00071 loss=0.2814 step/sec=5.405 | ETA 00:04:53 epoch=1 step=740 lr=0.00071 loss=0.3725 step/sec=5.491 | ETA 00:04:47 epoch=1 step=750 lr=0.00070 loss=0.2068 step/sec=5.443 | ETA 00:04:47 epoch=1 step=760 lr=0.00070 loss=0.0839 step/sec=5.409 | ETA 00:04:47 epoch=1 step=770 lr=0.00070 loss=0.1149 step/sec=5.441 | ETA 00:04:44 epoch=1 step=780 lr=0.00069 loss=0.2186 step/sec=5.449 | ETA 00:04:41 epoch=1 step=790 lr=0.00069 loss=0.1558 step/sec=5.358 | ETA 00:04:44 epoch=1 step=800 lr=0.00068 loss=0.1774 step/sec=5.440 | ETA 00:04:38 epoch=1 step=810 lr=0.00068 loss=0.1653 step/sec=5.441 | ETA 00:04:36 epoch=1 step=820 lr=0.00068 loss=0.1685 step/sec=5.320 | ETA 00:04:41 epoch=1 step=830 lr=0.00067 loss=0.2406 step/sec=5.330 | ETA 00:04:38 epoch=1 step=840 lr=0.00067 loss=0.2114 step/sec=4.642 | ETA 00:05:17 epoch=1 step=850 lr=0.00066 loss=0.0567 step/sec=4.909 | ETA 00:04:58 epoch=1 step=860 lr=0.00066 loss=0.2362 step/sec=4.899 | ETA 00:04:57 epoch=1 step=870 lr=0.00066 loss=0.2638 step/sec=4.954 | ETA 00:04:51 epoch=1 step=880 lr=0.00065 loss=0.2902 step/sec=4.766 | ETA 00:05:01 epoch=1 step=890 lr=0.00065 loss=0.1391 step/sec=4.997 | ETA 00:04:45 epoch=1 step=900 lr=0.00064 loss=0.2761 step/sec=5.202 | ETA 00:04:32 epoch=1 step=910 lr=0.00064 loss=0.2664 step/sec=5.476 | ETA 00:04:16 epoch=1 step=920 lr=0.00063 loss=0.1213 step/sec=5.614 | ETA 00:04:08 epoch=1 step=930 lr=0.00063 loss=0.2519 step/sec=5.749 | ETA 00:04:01 epoch=1 step=940 lr=0.00063 loss=0.3305 step/sec=5.545 | ETA 00:04:08 epoch=1 step=950 lr=0.00062 loss=0.1392 step/sec=5.580 | ETA 00:04:04 epoch=1 step=960 lr=0.00062 loss=0.3605 step/sec=5.415 | ETA 00:04:10 epoch=1 step=970 lr=0.00061 loss=0.1370 step/sec=5.743 | ETA 00:03:54 epoch=1 step=980 lr=0.00061 loss=0.1300 step/sec=5.769 | ETA 00:03:51 epoch=1 step=990 lr=0.00061 loss=0.1200 step/sec=5.744 | ETA 00:03:50 epoch=1 step=1000 lr=0.00060 loss=0.1760 step/sec=5.737 | ETA 00:03:49 epoch=1 step=1010 lr=0.00060 loss=0.1097 step/sec=5.438 | ETA 00:04:00 epoch=1 step=1020 lr=0.00059 loss=0.3858 step/sec=5.653 | ETA 00:03:49 epoch=1 step=1030 lr=0.00059 loss=0.1992 step/sec=5.750 | ETA 00:03:43 epoch=1 step=1040 lr=0.00059 loss=0.3470 step/sec=5.794 | ETA 00:03:40 epoch=1 step=1050 lr=0.00058 loss=0.3766 step/sec=5.709 | ETA 00:03:41 epoch=1 step=1060 lr=0.00058 loss=0.2608 step/sec=5.544 | ETA 00:03:46 epoch=1 step=1070 lr=0.00057 loss=0.0838 step/sec=5.548 | ETA 00:03:44 epoch=1 step=1080 lr=0.00057 loss=0.2124 step/sec=5.476 | ETA 00:03:45 epoch=1 step=1090 lr=0.00057 loss=0.2297 step/sec=5.793 | ETA 00:03:31 epoch=1 step=1100 lr=0.00056 loss=0.2067 step/sec=5.748 | ETA 00:03:31 epoch=1 step=1110 lr=0.00056 loss=0.0893 step/sec=5.695 | ETA 00:03:31 epoch=1 step=1120 lr=0.00055 loss=0.4034 step/sec=5.703 | ETA 00:03:29 epoch=1 step=1130 lr=0.00055 loss=0.1206 step/sec=5.772 | ETA 00:03:25 epoch=1 step=1140 lr=0.00054 loss=0.2643 step/sec=5.707 | ETA 00:03:26 epoch=1 step=1150 lr=0.00054 loss=0.2179 step/sec=5.586 | ETA 00:03:28 Save model checkpoint to ./saved_model/hrnet_optic/1 Evaluation start #Device count: 1 load test model: ./saved_model/hrnet_optic/1 Traceback (most recent call last): File "./pdseg/train.py", line 466, in main(args) File "./pdseg/train.py", line 453, in main train(cfg) File "./pdseg/train.py", line 409, in train use_gpu=args.use_gpu) File "/home/aistudio/pdseg/eval.py", line 140, in evaluate conf_mat.calculate(pred, grts, masks) File "/home/aistudio/pdseg/metrics.py", line 47, in calculate shape=(self.num_classes, self.num_classes)) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/scipy/sparse/compressed.py", line 57, in init other = self.class(coo_matrix(arg1, shape=shape)) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/scipy/sparse/coo.py", line 198, in init self._check() File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/scipy/sparse/coo.py", line 285, in _check raise ValueError('row index exceeds matrix dimensions') ValueError: row index exceeds matrix dimensions