{'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'BATCH_SIZE': 8, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'BACKBONE': 'xception_65', 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 101}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_CROP_SIZE': (769, 769)} #Device count: 2 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'BACKBONE': 'xception_65', 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 101}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #Device count: 2 #Train_Batch_Size: 4 batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet logits.type: {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'BACKBONE': 'xception_65', 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 101}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #Device count: 2 #Train_Batch_Size: 4 batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/240 step=10 lr=0.00100 loss=4.0772 step/sec=0.461 | ETA 107:23:14 epoch=1/240 step=20 lr=0.00100 loss=3.8854 step/sec=0.824 | ETA 60:05:27 epoch=1/240 step=30 lr=0.00100 loss=3.4220 step/sec=0.821 | ETA 60:20:33 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 4 #train_batch_size: 8 #batch_size_per_dev: 2 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 4 #train_batch_size: 8 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 Failed to find function: modeling.deeplabv3.deeplabv3 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 Failed to find function: modeling.deeplabv3.deeplabv3 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 Failed to find function: modeling.deeplabv3.deeplabv3 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 logits.type: {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 aux_logit: None logits.type: {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 logit.shape: (-1, 19, 769, 769) {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 logit.shape: (-1, 19, 769, 769) aux_logit.shape: (-1, 19, 769, 769) logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/240 step=10 lr=0.00100 loss=4.2020 step/sec=0.408 | ETA 121:31:01 epoch=1/240 step=20 lr=0.00100 loss=3.4331 step/sec=0.676 | ETA 73:13:53 epoch=1/240 step=30 lr=0.00100 loss=3.4738 step/sec=0.681 | ETA 72:45:16 epoch=1/240 step=40 lr=0.00100 loss=3.1761 step/sec=0.677 | ETA 73:07:05 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'xception_65', 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0], 'PSPNET': {'AuxHead': False, 'BACKBONE': 'hrnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_hrnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_hrnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/hrnet/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet Failed to find function: modeling.pspnet.pspnet {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'xception_65', 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0], 'PSPNET': {'AuxHead': False, 'BACKBONE': 'hrnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_hrnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_hrnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/hrnet/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet Failed to find function: modeling.pspnet.pspnet {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'xception_65', 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0], 'PSPNET': {'AuxHead': False, 'BACKBONE': 'hrnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_hrnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_hrnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/hrnet/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'xception_65', 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0], 'PSPNET': {'AuxHead': False, 'BACKBONE': 'hrnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_hrnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_hrnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/hrnet/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'xception_65', 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0], 'PSPNET': {'AuxHead': False, 'BACKBONE': 'hrnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_hrnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_hrnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/hrnet/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'FREEZE': {'MODEL_FILENAME': 'model', 'PARAMS_FILENAME': 'params', 'SAVE_DIR': 'freeze_model'}, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'xception_65', 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0], 'PSPNET': {'AuxHead': False, 'BACKBONE': 'hrnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 50}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_hrnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_hrnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/hrnet/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/hrnet/ not exists, training from scratch... Use multiprocess reader epoch=1/240 step=10 lr=0.00100 loss=2.7972 step/sec=0.314 | ETA 157:54:13 epoch=1/240 step=20 lr=0.00100 loss=1.8502 step/sec=0.758 | ETA 65:19:58 epoch=1/240 step=30 lr=0.00100 loss=1.4555 step/sec=0.756 | ETA 65:28:22 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'BACKBONE': 'xception_65', 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'LAYERS': 101, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'BACKBONE': 'resnet', 'DEPTH_MULTIPLIER': 1, 'LAYERS': 101}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/240 step=10 lr=0.00100 loss=3.7719 step/sec=0.303 | ETA 163:18:21 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/240 step=10 lr=0.00100 loss=3.7767 step/sec=0.365 | ETA 135:37:53 epoch=1/240 step=20 lr=0.00100 loss=3.4905 step/sec=0.568 | ETA 87:12:07 epoch=1/240 step=30 lr=0.00100 loss=2.7186 step/sec=0.567 | ETA 87:19:09 epoch=1/240 step=40 lr=0.00100 loss=3.2132 step/sec=0.559 | ETA 88:32:48 epoch=1/240 step=50 lr=0.00100 loss=2.8406 step/sec=0.567 | ETA 87:23:06 epoch=1/240 step=60 lr=0.00100 loss=2.9147 step/sec=0.566 | ETA 87:25:05 epoch=1/240 step=70 lr=0.00100 loss=2.9803 step/sec=0.565 | ETA 87:39:55 epoch=1/240 step=80 lr=0.00100 loss=2.5850 step/sec=0.563 | ETA 87:56:00 epoch=1/240 step=90 lr=0.00100 loss=2.4320 step/sec=0.565 | ETA 87:38:22 epoch=1/240 step=100 lr=0.00100 loss=2.7203 step/sec=0.569 | ETA 86:57:11 epoch=1/240 step=110 lr=0.00100 loss=2.5445 step/sec=0.566 | ETA 87:26:32 epoch=1/240 step=120 lr=0.00100 loss=2.7914 step/sec=0.565 | ETA 87:38:15 epoch=1/240 step=130 lr=0.00100 loss=3.1261 step/sec=0.563 | ETA 87:58:04 epoch=1/240 step=140 lr=0.00100 loss=2.6236 step/sec=0.559 | ETA 88:33:50 epoch=1/240 step=150 lr=0.00100 loss=2.2097 step/sec=0.571 | ETA 86:41:11 epoch=1/240 step=160 lr=0.00100 loss=2.5590 step/sec=0.563 | ETA 87:58:16 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'hrnet', 'BACKBONE_LAYERS': 101, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0], 'PSPNET': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_hrnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_hrnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/hrnet/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/hrnet/ not exists, training from scratch... Use multiprocess reader epoch=1/240 step=10 lr=0.00100 loss=2.7006 step/sec=0.340 | ETA 145:49:18 epoch=1/240 step=20 lr=0.00100 loss=2.5155 step/sec=0.736 | ETA 67:15:28 epoch=1/240 step=30 lr=0.00100 loss=1.9090 step/sec=0.758 | ETA 65:21:34 epoch=1/240 step=40 lr=0.00100 loss=2.6938 step/sec=0.754 | ETA 65:40:32 epoch=1/240 step=50 lr=0.00100 loss=1.8135 step/sec=0.752 | ETA 65:53:28 epoch=1/240 step=60 lr=0.00100 loss=2.1372 step/sec=0.741 | ETA 66:51:26 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1, 'ENABLE_DECODER': True, 'ENCODER_WITH_ASPP': True, 'OUTPUT_STRIDE': 16}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/240 step=10 lr=0.00100 loss=4.3662 step/sec=0.405 | ETA 122:09:14 epoch=1/240 step=20 lr=0.00100 loss=3.4859 step/sec=0.666 | ETA 74:23:32 epoch=1/240 step=30 lr=0.00100 loss=3.7232 step/sec=0.662 | ETA 74:46:46 epoch=1/240 step=40 lr=0.00100 loss=3.4560 step/sec=0.667 | ETA 74:13:01 epoch=1/240 step=50 lr=0.00100 loss=2.7223 step/sec=0.665 | ETA 74:30:51 epoch=1/240 step=60 lr=0.00100 loss=2.7973 step/sec=0.663 | ETA 74:41:53 epoch=1/240 step=70 lr=0.00100 loss=3.0885 step/sec=0.666 | ETA 74:21:27 epoch=1/240 step=80 lr=0.00100 loss=2.7773 step/sec=0.662 | ETA 74:49:45 epoch=1/240 step=90 lr=0.00100 loss=2.5828 step/sec=0.669 | ETA 74:02:20 epoch=1/240 step=100 lr=0.00100 loss=2.9299 step/sec=0.662 | ETA 74:43:54 epoch=1/240 step=110 lr=0.00100 loss=2.4559 step/sec=0.666 | ETA 74:17:35 epoch=1/240 step=120 lr=0.00100 loss=2.8510 step/sec=0.660 | ETA 75:02:43 epoch=1/240 step=130 lr=0.00100 loss=3.2617 step/sec=0.668 | ETA 74:03:00 epoch=1/240 step=140 lr=0.00100 loss=2.5842 step/sec=0.664 | ETA 74:32:14 epoch=1/240 step=150 lr=0.00100 loss=2.5723 step/sec=0.661 | ETA 74:55:10 epoch=1/240 step=160 lr=0.00100 loss=2.4573 step/sec=0.662 | ETA 74:47:30 epoch=1/240 step=170 lr=0.00100 loss=2.2364 step/sec=0.662 | ETA 74:44:42 epoch=1/240 step=180 lr=0.00100 loss=2.7714 step/sec=0.666 | ETA 74:18:36 epoch=1/240 step=190 lr=0.00100 loss=2.9037 step/sec=0.666 | ETA 74:15:44 epoch=1/240 step=200 lr=0.00100 loss=2.6817 step/sec=0.659 | ETA 75:02:51 epoch=1/240 step=210 lr=0.00100 loss=2.5190 step/sec=0.664 | ETA 74:28:52 epoch=1/240 step=220 lr=0.00100 loss=2.4552 step/sec=0.662 | ETA 74:46:09 epoch=1/240 step=230 lr=0.00100 loss=2.4132 step/sec=0.668 | ETA 74:06:19 epoch=1/240 step=240 lr=0.00100 loss=2.6188 step/sec=0.665 | ETA 74:25:25 epoch=1/240 step=250 lr=0.00100 loss=2.6648 step/sec=0.666 | ETA 74:19:23 epoch=1/240 step=260 lr=0.00100 loss=2.2486 step/sec=0.672 | ETA 73:35:07 epoch=1/240 step=270 lr=0.00100 loss=2.4010 step/sec=0.658 | ETA 75:08:58 epoch=1/240 step=280 lr=0.00100 loss=2.4540 step/sec=0.670 | ETA 73:48:14 epoch=1/240 step=290 lr=0.00100 loss=2.2084 step/sec=0.661 | ETA 74:47:37 epoch=1/240 step=300 lr=0.00100 loss=2.4844 step/sec=0.663 | ETA 74:33:51 epoch=1/240 step=310 lr=0.00100 loss=3.0036 step/sec=0.665 | ETA 74:24:14 epoch=1/240 step=320 lr=0.00100 loss=2.7633 step/sec=0.660 | ETA 74:52:16 epoch=1/240 step=330 lr=0.00100 loss=2.3222 step/sec=0.665 | ETA 74:21:37 epoch=1/240 step=340 lr=0.00100 loss=2.2641 step/sec=0.665 | ETA 74:20:05 epoch=1/240 step=350 lr=0.00100 loss=2.4916 step/sec=0.661 | ETA 74:45:56 epoch=1/240 step=360 lr=0.00100 loss=2.4329 step/sec=0.663 | ETA 74:36:52 epoch=1/240 step=370 lr=0.00100 loss=2.3110 step/sec=0.659 | ETA 75:01:11 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/240 step=10 lr=0.00100 loss=4.4750 step/sec=0.350 | ETA 141:38:06 epoch=1/240 step=20 lr=0.00100 loss=3.5411 step/sec=0.549 | ETA 90:15:33 epoch=1/240 step=30 lr=0.00100 loss=3.5141 step/sec=0.551 | ETA 89:48:25 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 56, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 48, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 3, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/3 step=10 lr=0.00081 loss=4.1386 step/sec=0.450 | ETA 00:01:11 Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ epoch=2/3 step=20 lr=0.00057 loss=3.3114 step/sec=0.646 | ETA 00:00:34 Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ epoch=3/3 step=30 lr=0.00032 loss=3.2669 step/sec=0.618 | ETA 00:00:19 epoch=3/3 step=40 lr=0.00006 loss=3.2028 step/sec=0.833 | ETA 00:00:02 Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 56, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 48, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 3, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/3 step=2 lr=0.00098 loss=4.4368 step/sec=0.167 | ETA 00:03:59 epoch=1/3 step=4 lr=0.00094 loss=4.2521 step/sec=0.807 | ETA 00:00:47 epoch=1/3 step=6 lr=0.00089 loss=3.9196 step/sec=0.760 | ETA 00:00:47 epoch=1/3 step=8 lr=0.00085 loss=4.0833 step/sec=0.671 | ETA 00:00:50 epoch=1/3 step=10 lr=0.00081 loss=4.1826 step/sec=0.841 | ETA 00:00:38 epoch=1/3 step=12 lr=0.00077 loss=3.5480 step/sec=0.814 | ETA 00:00:36 epoch=1/3 step=14 lr=0.00072 loss=3.7976 step/sec=0.593 | ETA 00:00:47 Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ epoch=2/3 step=16 lr=0.00066 loss=3.7694 step/sec=0.276 | ETA 00:01:34 epoch=2/3 step=18 lr=0.00061 loss=3.6361 step/sec=0.835 | ETA 00:00:28 epoch=2/3 step=20 lr=0.00057 loss=3.5078 step/sec=0.725 | ETA 00:00:30 epoch=2/3 step=22 lr=0.00052 loss=3.7751 step/sec=0.843 | ETA 00:00:23 epoch=2/3 step=24 lr=0.00048 loss=2.9981 step/sec=0.833 | ETA 00:00:21 epoch=2/3 step=26 lr=0.00043 loss=3.5111 step/sec=0.641 | ETA 00:00:24 epoch=2/3 step=28 lr=0.00039 loss=3.0423 step/sec=0.841 | ETA 00:00:16 Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ epoch=3/3 step=30 lr=0.00032 loss=3.4324 step/sec=0.271 | ETA 00:00:44 epoch=3/3 step=32 lr=0.00027 loss=3.9362 step/sec=0.835 | ETA 00:00:11 epoch=3/3 step=34 lr=0.00022 loss=2.6271 step/sec=0.824 | ETA 00:00:09 epoch=3/3 step=36 lr=0.00017 loss=3.3786 step/sec=0.647 | ETA 00:00:09 epoch=3/3 step=38 lr=0.00012 loss=2.8715 step/sec=0.838 | ETA 00:00:04 epoch=3/3 step=40 lr=0.00006 loss=3.4629 step/sec=0.764 | ETA 00:00:02 epoch=3/3 step=42 lr=0.00000 loss=3.4207 step/sec=0.657 | ETA 00:00:00 Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 56, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 48, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 3, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/3 step=2 lr=0.00098 loss=4.1013 step/sec=0.143 | ETA 00:04:39 epoch=1/3 step=4 lr=0.00094 loss=4.1724 step/sec=0.451 | ETA 00:01:24 epoch=1/3 step=6 lr=0.00089 loss=4.0306 step/sec=0.451 | ETA 00:01:19 epoch=1/3 step=8 lr=0.00085 loss=4.3476 step/sec=0.449 | ETA 00:01:15 epoch=1/3 step=10 lr=0.00081 loss=3.9499 step/sec=0.456 | ETA 00:01:10 epoch=1/3 step=12 lr=0.00077 loss=4.0282 step/sec=0.455 | ETA 00:01:05 epoch=1/3 step=14 lr=0.00072 loss=3.9755 step/sec=0.456 | ETA 00:01:01 Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ epoch=2/3 step=16 lr=0.00066 loss=3.1431 step/sec=0.229 | ETA 00:01:53 epoch=2/3 step=18 lr=0.00061 loss=3.8308 step/sec=0.453 | ETA 00:00:52 epoch=2/3 step=20 lr=0.00057 loss=3.3716 step/sec=0.448 | ETA 00:00:49 epoch=2/3 step=22 lr=0.00052 loss=3.5819 step/sec=0.450 | ETA 00:00:44 epoch=2/3 step=24 lr=0.00048 loss=3.2173 step/sec=0.455 | ETA 00:00:39 epoch=2/3 step=26 lr=0.00043 loss=3.6328 step/sec=0.455 | ETA 00:00:35 epoch=2/3 step=28 lr=0.00039 loss=3.5825 step/sec=0.453 | ETA 00:00:30 Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ epoch=3/3 step=30 lr=0.00032 loss=3.7638 step/sec=0.222 | ETA 00:00:54 epoch=3/3 step=32 lr=0.00027 loss=3.2656 step/sec=0.441 | ETA 00:00:22 epoch=3/3 step=34 lr=0.00022 loss=3.0186 step/sec=0.451 | ETA 00:00:17 epoch=3/3 step=36 lr=0.00017 loss=3.1815 step/sec=0.458 | ETA 00:00:13 epoch=3/3 step=38 lr=0.00012 loss=3.2035 step/sec=0.456 | ETA 00:00:08 epoch=3/3 step=40 lr=0.00006 loss=3.4582 step/sec=0.453 | ETA 00:00:04 epoch=3/3 step=42 lr=0.00000 loss=3.2495 step/sec=0.455 | ETA 00:00:00 Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 56, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 48, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'hrnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0], 'PSPNET': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 240, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_hrnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_hrnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/hrnet/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/hrnet/ not exists, training from scratch... Use multiprocess reader epoch=1/240 step=2 lr=0.00100 loss=3.2843 step/sec=0.094 | ETA 09:56:27 epoch=1/240 step=4 lr=0.00100 loss=3.2916 step/sec=0.744 | ETA 01:15:09 epoch=1/240 step=6 lr=0.00100 loss=3.1003 step/sec=0.714 | ETA 01:18:18 epoch=1/240 step=8 lr=0.00100 loss=2.5840 step/sec=0.752 | ETA 01:14:14 epoch=1/240 step=10 lr=0.00100 loss=2.8527 step/sec=0.735 | ETA 01:15:57 epoch=1/240 step=12 lr=0.00100 loss=2.1349 step/sec=0.756 | ETA 01:13:46 epoch=1/240 step=14 lr=0.00100 loss=3.2585 step/sec=0.767 | ETA 01:12:43 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=2/240 step=16 lr=0.00100 loss=2.0753 step/sec=0.280 | ETA 03:19:05 epoch=2/240 step=18 lr=0.00100 loss=2.9004 step/sec=0.748 | ETA 01:14:29 epoch=2/240 step=20 lr=0.00099 loss=3.0689 step/sec=0.737 | ETA 01:15:30 epoch=2/240 step=22 lr=0.00099 loss=1.5404 step/sec=0.752 | ETA 01:13:59 epoch=2/240 step=24 lr=0.00099 loss=1.6785 step/sec=0.674 | ETA 01:22:31 epoch=2/240 step=26 lr=0.00099 loss=2.3259 step/sec=0.731 | ETA 01:16:01 epoch=2/240 step=28 lr=0.00099 loss=1.6581 step/sec=0.766 | ETA 01:12:32 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=3/240 step=30 lr=0.00099 loss=2.5944 step/sec=0.272 | ETA 03:24:11 epoch=3/240 step=32 lr=0.00099 loss=1.5762 step/sec=0.740 | ETA 01:14:56 epoch=3/240 step=34 lr=0.00099 loss=1.3123 step/sec=0.732 | ETA 01:15:41 epoch=3/240 step=36 lr=0.00099 loss=2.5277 step/sec=0.738 | ETA 01:15:02 epoch=3/240 step=38 lr=0.00099 loss=2.2207 step/sec=0.769 | ETA 01:11:59 epoch=3/240 step=40 lr=0.00099 loss=1.5792 step/sec=0.716 | ETA 01:17:14 epoch=3/240 step=42 lr=0.00099 loss=2.2094 step/sec=0.764 | ETA 01:12:24 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=4/240 step=44 lr=0.00099 loss=1.8894 step/sec=0.286 | ETA 03:12:54 epoch=4/240 step=46 lr=0.00099 loss=1.7511 step/sec=0.729 | ETA 01:15:44 epoch=4/240 step=48 lr=0.00099 loss=2.1341 step/sec=0.712 | ETA 01:17:31 epoch=4/240 step=50 lr=0.00099 loss=2.0900 step/sec=0.720 | ETA 01:16:36 epoch=4/240 step=52 lr=0.00099 loss=2.0653 step/sec=0.589 | ETA 01:33:32 epoch=4/240 step=54 lr=0.00099 loss=1.9960 step/sec=0.682 | ETA 01:20:48 epoch=4/240 step=56 lr=0.00098 loss=1.9788 step/sec=0.719 | ETA 01:16:35 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=5/240 step=58 lr=0.00098 loss=1.6586 step/sec=0.285 | ETA 03:12:47 epoch=5/240 step=60 lr=0.00098 loss=1.6079 step/sec=0.755 | ETA 01:12:48 epoch=5/240 step=62 lr=0.00098 loss=1.7617 step/sec=0.736 | ETA 01:14:40 epoch=5/240 step=64 lr=0.00098 loss=1.4224 step/sec=0.706 | ETA 01:17:45 epoch=5/240 step=66 lr=0.00098 loss=2.1521 step/sec=0.751 | ETA 01:13:03 epoch=5/240 step=68 lr=0.00098 loss=1.9026 step/sec=0.770 | ETA 01:11:17 epoch=5/240 step=70 lr=0.00098 loss=1.7770 step/sec=0.772 | ETA 01:11:03 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=6/240 step=72 lr=0.00098 loss=1.4976 step/sec=0.311 | ETA 02:56:03 epoch=6/240 step=74 lr=0.00098 loss=1.7116 step/sec=0.691 | ETA 01:19:12 epoch=6/240 step=76 lr=0.00098 loss=1.8032 step/sec=0.591 | ETA 01:32:35 epoch=6/240 step=78 lr=0.00098 loss=2.0788 step/sec=0.761 | ETA 01:11:52 epoch=6/240 step=80 lr=0.00098 loss=1.4603 step/sec=0.730 | ETA 01:14:53 epoch=6/240 step=82 lr=0.00098 loss=1.7331 step/sec=0.737 | ETA 01:14:06 epoch=6/240 step=84 lr=0.00098 loss=1.8485 step/sec=0.763 | ETA 01:11:36 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=7/240 step=86 lr=0.00098 loss=1.4056 step/sec=0.276 | ETA 03:17:43 epoch=7/240 step=88 lr=0.00098 loss=1.3731 step/sec=0.756 | ETA 01:12:08 epoch=7/240 step=90 lr=0.00098 loss=1.2093 step/sec=0.744 | ETA 01:13:16 epoch=7/240 step=92 lr=0.00098 loss=1.6703 step/sec=0.753 | ETA 01:12:21 epoch=7/240 step=94 lr=0.00097 loss=1.6043 step/sec=0.743 | ETA 01:13:15 epoch=7/240 step=96 lr=0.00097 loss=1.2910 step/sec=0.750 | ETA 01:12:31 epoch=7/240 step=98 lr=0.00097 loss=1.7411 step/sec=0.752 | ETA 01:12:20 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=8/240 step=100 lr=0.00097 loss=1.2600 step/sec=0.310 | ETA 02:55:25 epoch=8/240 step=102 lr=0.00097 loss=1.5772 step/sec=0.751 | ETA 01:12:18 epoch=8/240 step=104 lr=0.00097 loss=1.4286 step/sec=0.755 | ETA 01:11:54 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 56, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 48, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'hrnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0], 'PSPNET': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 4, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_hrnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_hrnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/hrnet/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/hrnet/ not exists, training from scratch... Use multiprocess reader epoch=1/4 step=2 lr=0.00098 loss=3.2141 step/sec=0.099 | ETA 00:09:05 epoch=1/4 step=4 lr=0.00095 loss=3.1031 step/sec=0.747 | ETA 00:01:09 epoch=1/4 step=6 lr=0.00092 loss=2.3773 step/sec=0.758 | ETA 00:01:05 epoch=1/4 step=8 lr=0.00089 loss=2.6201 step/sec=0.750 | ETA 00:01:04 epoch=1/4 step=10 lr=0.00086 loss=2.8249 step/sec=0.757 | ETA 00:01:00 epoch=1/4 step=12 lr=0.00082 loss=3.4406 step/sec=0.763 | ETA 00:00:57 epoch=1/4 step=14 lr=0.00079 loss=2.4824 step/sec=0.756 | ETA 00:00:55 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=2/4 step=16 lr=0.00074 loss=1.7141 step/sec=0.293 | ETA 00:02:16 epoch=2/4 step=18 lr=0.00071 loss=1.5779 step/sec=0.730 | ETA 00:00:52 epoch=2/4 step=20 lr=0.00068 loss=1.6781 step/sec=0.730 | ETA 00:00:49 epoch=2/4 step=22 lr=0.00064 loss=2.4676 step/sec=0.755 | ETA 00:00:45 epoch=2/4 step=24 lr=0.00061 loss=2.9528 step/sec=0.764 | ETA 00:00:41 epoch=2/4 step=26 lr=0.00058 loss=1.9822 step/sec=0.757 | ETA 00:00:39 epoch=2/4 step=28 lr=0.00054 loss=1.6825 step/sec=0.771 | ETA 00:00:36 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=3/4 step=30 lr=0.00049 loss=2.3975 step/sec=0.278 | ETA 00:01:33 epoch=3/4 step=32 lr=0.00046 loss=2.6615 step/sec=0.742 | ETA 00:00:32 epoch=3/4 step=34 lr=0.00042 loss=2.3163 step/sec=0.690 | ETA 00:00:31 epoch=3/4 step=36 lr=0.00039 loss=1.8193 step/sec=0.734 | ETA 00:00:27 epoch=3/4 step=38 lr=0.00035 loss=1.6377 step/sec=0.762 | ETA 00:00:23 epoch=3/4 step=40 lr=0.00032 loss=1.6747 step/sec=0.764 | ETA 00:00:20 epoch=3/4 step=42 lr=0.00028 loss=1.6103 step/sec=0.770 | ETA 00:00:18 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=4/4 step=44 lr=0.00023 loss=1.5491 step/sec=0.303 | ETA 00:00:39 epoch=4/4 step=46 lr=0.00019 loss=2.8150 step/sec=0.763 | ETA 00:00:13 epoch=4/4 step=48 lr=0.00015 loss=1.6941 step/sec=0.754 | ETA 00:00:10 epoch=4/4 step=50 lr=0.00011 loss=1.6356 step/sec=0.763 | ETA 00:00:07 epoch=4/4 step=52 lr=0.00007 loss=1.6909 step/sec=0.761 | ETA 00:00:05 epoch=4/4 step=54 lr=0.00003 loss=1.5791 step/sec=0.760 | ETA 00:00:02 epoch=4/4 step=56 lr=0.00000 loss=1.3418 step/sec=0.765 | ETA 00:00:00 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 56, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 48, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 4, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'hrnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0], 'PSPNET': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 4, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_hrnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_hrnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/hrnet/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/hrnet/ not exists, training from scratch... Use multiprocess reader epoch=1/4 step=2 lr=0.00098 loss=3.5025 step/sec=0.100 | ETA 00:09:00 epoch=1/4 step=4 lr=0.00095 loss=3.1647 step/sec=0.759 | ETA 00:01:08 epoch=1/4 step=6 lr=0.00092 loss=2.7648 step/sec=0.742 | ETA 00:01:07 epoch=1/4 step=8 lr=0.00089 loss=2.1560 step/sec=0.725 | ETA 00:01:06 epoch=1/4 step=10 lr=0.00086 loss=2.3052 step/sec=0.758 | ETA 00:01:00 epoch=1/4 step=12 lr=0.00082 loss=3.2465 step/sec=0.766 | ETA 00:00:57 epoch=1/4 step=14 lr=0.00079 loss=2.2845 step/sec=0.766 | ETA 00:00:54 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=2/4 step=16 lr=0.00074 loss=3.2021 step/sec=0.266 | ETA 00:02:30 epoch=2/4 step=18 lr=0.00071 loss=3.1324 step/sec=0.743 | ETA 00:00:51 epoch=2/4 step=20 lr=0.00068 loss=2.7344 step/sec=0.756 | ETA 00:00:47 epoch=2/4 step=22 lr=0.00064 loss=2.2053 step/sec=0.735 | ETA 00:00:46 epoch=2/4 step=24 lr=0.00061 loss=2.6966 step/sec=0.762 | ETA 00:00:42 epoch=2/4 step=26 lr=0.00058 loss=1.8520 step/sec=0.735 | ETA 00:00:40 epoch=2/4 step=28 lr=0.00054 loss=2.2937 step/sec=0.759 | ETA 00:00:36 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=3/4 step=30 lr=0.00049 loss=2.5425 step/sec=0.259 | ETA 00:01:40 epoch=3/4 step=32 lr=0.00046 loss=1.8992 step/sec=0.679 | ETA 00:00:35 epoch=3/4 step=34 lr=0.00042 loss=2.1873 step/sec=0.769 | ETA 00:00:28 epoch=3/4 step=36 lr=0.00039 loss=1.7678 step/sec=0.735 | ETA 00:00:27 epoch=3/4 step=38 lr=0.00035 loss=2.1245 step/sec=0.761 | ETA 00:00:23 epoch=3/4 step=40 lr=0.00032 loss=1.7823 step/sec=0.739 | ETA 00:00:21 epoch=3/4 step=42 lr=0.00028 loss=1.6311 step/sec=0.753 | ETA 00:00:18 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ epoch=4/4 step=44 lr=0.00023 loss=1.7080 step/sec=0.301 | ETA 00:00:39 epoch=4/4 step=46 lr=0.00019 loss=2.0921 step/sec=0.736 | ETA 00:00:13 epoch=4/4 step=48 lr=0.00015 loss=2.1906 step/sec=0.747 | ETA 00:00:10 epoch=4/4 step=50 lr=0.00011 loss=1.6024 step/sec=0.744 | ETA 00:00:08 epoch=4/4 step=52 lr=0.00007 loss=2.5127 step/sec=0.755 | ETA 00:00:05 epoch=4/4 step=54 lr=0.00003 loss=1.4936 step/sec=0.752 | ETA 00:00:02 epoch=4/4 step=56 lr=0.00000 loss=1.8734 step/sec=0.758 | ETA 00:00:00 Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ Save model checkpoint to snapshots/pspnet_hrnet_cityscapes/ {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 56, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 48, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 4, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_res101_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_res101_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/4 step=2 lr=0.00098 loss=4.2775 step/sec=0.156 | ETA 00:05:46 epoch=1/4 step=4 lr=0.00095 loss=4.1790 step/sec=0.539 | ETA 00:01:36 epoch=1/4 step=6 lr=0.00092 loss=4.0308 step/sec=0.550 | ETA 00:01:30 epoch=1/4 step=8 lr=0.00089 loss=4.1610 step/sec=0.552 | ETA 00:01:27 epoch=1/4 step=10 lr=0.00086 loss=3.7599 step/sec=0.550 | ETA 00:01:23 epoch=1/4 step=12 lr=0.00082 loss=3.5398 step/sec=0.552 | ETA 00:01:19 epoch=1/4 step=14 lr=0.00079 loss=3.7867 step/sec=0.553 | ETA 00:01:15 epoch=2/4 step=16 lr=0.00074 loss=3.2999 step/sec=0.298 | ETA 00:02:14 epoch=2/4 step=18 lr=0.00071 loss=3.8573 step/sec=0.547 | ETA 00:01:09 epoch=2/4 step=20 lr=0.00068 loss=3.5154 step/sec=0.539 | ETA 00:01:06 epoch=2/4 step=22 lr=0.00064 loss=2.8900 step/sec=0.546 | ETA 00:01:02 epoch=2/4 step=24 lr=0.00061 loss=3.4460 step/sec=0.547 | ETA 00:00:58 epoch=2/4 step=26 lr=0.00058 loss=2.8036 step/sec=0.552 | ETA 00:00:54 epoch=2/4 step=28 lr=0.00054 loss=2.8800 step/sec=0.550 | ETA 00:00:50 epoch=3/4 step=30 lr=0.00049 loss=3.6902 step/sec=0.287 | ETA 00:01:30 epoch=3/4 step=32 lr=0.00046 loss=3.3163 step/sec=0.545 | ETA 00:00:44 epoch=3/4 step=34 lr=0.00042 loss=2.6073 step/sec=0.545 | ETA 00:00:40 epoch=3/4 step=36 lr=0.00039 loss=2.7500 step/sec=0.538 | ETA 00:00:37 epoch=3/4 step=38 lr=0.00035 loss=3.2859 step/sec=0.550 | ETA 00:00:32 epoch=3/4 step=40 lr=0.00032 loss=2.8090 step/sec=0.550 | ETA 00:00:29 epoch=3/4 step=42 lr=0.00028 loss=3.2376 step/sec=0.552 | ETA 00:00:25 epoch=4/4 step=44 lr=0.00023 loss=3.5384 step/sec=0.277 | ETA 00:00:43 epoch=4/4 step=46 lr=0.00019 loss=3.5167 step/sec=0.537 | ETA 00:00:18 epoch=4/4 step=48 lr=0.00015 loss=3.2708 step/sec=0.550 | ETA 00:00:14 epoch=4/4 step=50 lr=0.00011 loss=2.8495 step/sec=0.550 | ETA 00:00:10 epoch=4/4 step=52 lr=0.00007 loss=2.7998 step/sec=0.550 | ETA 00:00:07 epoch=4/4 step=54 lr=0.00003 loss=2.9718 step/sec=0.548 | ETA 00:00:03 epoch=4/4 step=56 lr=0.00000 loss=2.4637 step/sec=0.549 | ETA 00:00:00 Save model checkpoint to ./snapshots/deeplabv3_res101_cityscapes/ {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 56, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 48, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 4, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 56, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 48, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.001, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 4, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 0.0001}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': './snapshots/deeplabv3_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 2, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/4 step=2 lr=0.00098 loss=4.3116 step/sec=0.138 | ETA 00:06:32 epoch=1/4 step=4 lr=0.00095 loss=4.3726 step/sec=0.578 | ETA 00:01:29 epoch=1/4 step=6 lr=0.00092 loss=4.1660 step/sec=0.554 | ETA 00:01:30 epoch=1/4 step=8 lr=0.00089 loss=3.9048 step/sec=0.555 | ETA 00:01:26 epoch=1/4 step=10 lr=0.00086 loss=3.7001 step/sec=0.550 | ETA 00:01:23 epoch=1/4 step=12 lr=0.00082 loss=3.9580 step/sec=0.551 | ETA 00:01:19 epoch=1/4 step=14 lr=0.00079 loss=3.8919 step/sec=0.553 | ETA 00:01:15 Save model checkpoint to ./snapshots/deeplabv3_resnet_cityscapes/ epoch=2/4 step=16 lr=0.00074 loss=4.1291 step/sec=0.230 | ETA 00:02:53 epoch=2/4 step=18 lr=0.00071 loss=2.9428 step/sec=0.534 | ETA 00:01:11 epoch=2/4 step=20 lr=0.00068 loss=3.0347 step/sec=0.547 | ETA 00:01:05 epoch=2/4 step=22 lr=0.00064 loss=3.3946 step/sec=0.550 | ETA 00:01:01 epoch=2/4 step=24 lr=0.00061 loss=3.8066 step/sec=0.543 | ETA 00:00:58 epoch=2/4 step=26 lr=0.00058 loss=3.2188 step/sec=0.551 | ETA 00:00:54 epoch=2/4 step=28 lr=0.00054 loss=3.5722 step/sec=0.551 | ETA 00:00:50 Save model checkpoint to ./snapshots/deeplabv3_resnet_cityscapes/ epoch=3/4 step=30 lr=0.00051 loss=3.1722 step/sec=0.268 | ETA 00:01:36 epoch=3/4 step=32 lr=0.00048 loss=3.7604 step/sec=0.544 | ETA 00:00:44 epoch=3/4 step=34 lr=0.00044 loss=3.3976 step/sec=0.547 | ETA 00:00:40 epoch=3/4 step=36 lr=0.00041 loss=3.3514 step/sec=0.544 | ETA 00:00:36 epoch=3/4 step=38 lr=0.00037 loss=3.3258 step/sec=0.543 | ETA 00:00:33 epoch=3/4 step=40 lr=0.00034 loss=2.9805 step/sec=0.551 | ETA 00:00:29 epoch=3/4 step=42 lr=0.00030 loss=3.1827 step/sec=0.550 | ETA 00:00:25 Save model checkpoint to ./snapshots/deeplabv3_resnet_cityscapes/ epoch=4/4 step=44 lr=0.00026 loss=2.3779 step/sec=0.261 | ETA 00:00:46 epoch=4/4 step=46 lr=0.00023 loss=3.1238 step/sec=0.534 | ETA 00:00:18 epoch=4/4 step=48 lr=0.00019 loss=2.9242 step/sec=0.535 | ETA 00:00:14 epoch=4/4 step=50 lr=0.00015 loss=2.9668 step/sec=0.559 | ETA 00:00:10 epoch=4/4 step=52 lr=0.00011 loss=2.7224 step/sec=0.542 | ETA 00:00:07 epoch=4/4 step=54 lr=0.00007 loss=3.3162 step/sec=0.549 | ETA 00:00:03 epoch=4/4 step=56 lr=0.00003 loss=3.0046 step/sec=0.549 | ETA 00:00:00 Save model checkpoint to ./snapshots/deeplabv3_resnet_cityscapes/ Save model checkpoint to ./snapshots/deeplabv3_resnet_cityscapes/ {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ Save model checkpoint to snapshots/pspnet_resnet_cityscapes/ {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multi-thread reader preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() img.type: grt.type: input_shape [training mode]: (769, 769, 3) img.type: grt.type: preparing entering self.process_image() epoch=1/80 step=5/59440 lr=0.01000 loss=4.2021 step/sec=0.101 | ETA 162:59:23 {'AUG': {'AUG_METHOD': 'stepscaling', 'FIX_RESIZE_SIZE': (640, 640), 'FLIP': True, 'FLIP_RATIO': 0.5, 'INF_RESIZE_VALUE': 500, 'MAX_RESIZE_VALUE': 600, 'MAX_SCALE_FACTOR': 2.0, 'MIN_RESIZE_VALUE': 400, 'MIN_SCALE_FACTOR': 0.5, 'MIRROR': True, 'RICH_CROP': {'ASPECT_RATIO': 0.33, 'BLUR': True, 'BLUR_RATIO': 0.1, 'BRIGHTNESS_JITTER_RATIO': 0.5, 'CONTRAST_JITTER_RATIO': 0.5, 'ENABLE': True, 'MAX_ROTATION': 15, 'MIN_AREA_RATIO': 0.5, 'SATURATION_JITTER_RATIO': 0.5}, 'SCALE_STEP_SIZE': 0.25}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multi-thread reader epoch=1/80 step=5/59440 lr=0.01000 loss=3.9238 step/sec=0.100 | ETA 164:25:02 epoch=1/80 step=10/59440 lr=0.01000 loss=3.4924 step/sec=0.119 | ETA 138:08:44 {'DATAAUG': {'BASE_SIZE': 1024, 'CROP_SIZE': 769, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.75}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multi-thread reader epoch=1/80 step=5/59440 lr=0.01000 loss=3.4908 step/sec=0.101 | ETA 163:04:53 epoch=1/80 step=10/59440 lr=0.01000 loss=3.2548 step/sec=0.120 | ETA 137:40:52 epoch=1/80 step=15/59440 lr=0.01000 loss=2.6542 step/sec=0.120 | ETA 137:10:48 {'DATAAUG': {'BASE_SIZE': 1024, 'CROP_SIZE': 769, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.75}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'pspnet', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/pspnet_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/pspnet_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 2, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 1 #train_batch_size: 2 #batch_size_per_dev: 2 func_name: modeling.pspnet.pspnet module_name: src.models.modeling.pspnet logits.type: Sync BatchNorm strategy will not be effective if GPU device count <= 1 Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multi-thread reader epoch=1/80 step=10/118960 lr=0.01000 loss=3.4338 step/sec=0.233 | ETA 142:06:43 epoch=1/80 step=20/118960 lr=0.01000 loss=3.0817 step/sec=0.240 | ETA 137:28:18 epoch=1/80 step=30/118960 lr=0.01000 loss=2.3315 step/sec=0.240 | ETA 137:50:11 {'DATAAUG': {'BASE_SIZE': 1024, 'CROP_SIZE': 769, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.75}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 1, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 1 #train_batch_size: 1 #batch_size_per_dev: 1 func_name: modeling.glore.glore module_name: src.models.modeling.glore Failed to find function: modeling.glore.glore {'DATAAUG': {'BASE_SIZE': 1024, 'CROP_SIZE': 769, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.75}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 1, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 1 #train_batch_size: 1 #batch_size_per_dev: 1 func_name: modeling.glore.glore module_name: src.models.modeling.glore {'DATAAUG': {'BASE_SIZE': 1024, 'CROP_SIZE': 769, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.75}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 1, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 1 #train_batch_size: 1 #batch_size_per_dev: 1 func_name: modeling.glore.glore module_name: src.models.modeling.glore {'DATAAUG': {'BASE_SIZE': 1024, 'CROP_SIZE': 769, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.75}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 1, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 1 #train_batch_size: 1 #batch_size_per_dev: 1 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 9409) {'DATAAUG': {'BASE_SIZE': 1024, 'CROP_SIZE': 769, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.75}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (2049, 1025), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 1, 'TRAIN_CROP_SIZE': (769, 769)} #device count: 1 #train_batch_size: 1 #batch_size_per_dev: 1 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 9409) logits.type: Sync BatchNorm strategy will not be effective if GPU device count <= 1 Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multi-thread reader epoch=1/80 step=10/238000 lr=0.01000 loss=3.8882 step/sec=0.461 | ETA 143:28:26 epoch=1/80 step=20/238000 lr=0.01000 loss=2.5311 step/sec=0.494 | ETA 133:47:39 {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.005, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_pascalContext/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 1, 'TRAIN_CROP_SIZE': (520, 520)} {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.005, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_pascalContext/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 1, 'TRAIN_CROP_SIZE': (520, 520)} #device count: 1 #train_batch_size: 1 #batch_size_per_dev: 1 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 4225) logits.type: Sync BatchNorm strategy will not be effective if GPU device count <= 1 Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader Save model checkpoint to snapshots/glore_res101_pascalContext/ Save model checkpoint to snapshots/glore_res101_pascalContext/ Save model checkpoint to snapshots/glore_res101_pascalContext/ Save model checkpoint to snapshots/glore_res101_pascalContext/ Save model checkpoint to snapshots/glore_res101_pascalContext/ Save model checkpoint to snapshots/glore_res101_pascalContext/ Save model checkpoint to snapshots/glore_res101_pascalContext/ Save model checkpoint to snapshots/glore_res101_pascalContext/ Save model checkpoint to snapshots/glore_res101_pascalContext/ {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.005, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_pascalContext/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 1, 'TRAIN_CROP_SIZE': (520, 520)} {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.005, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_pascalContext/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 1, 'TRAIN_CROP_SIZE': (520, 520)} #device count: 1 #train_batch_size: 1 #batch_size_per_dev: 1 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 4225) logits.type: Sync BatchNorm strategy will not be effective if GPU device count <= 1 Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader epoch=6/80 step=10/399840 lr=0.00500 loss=6.3172 step/sec=3.075 | ETA 36:07:23 epoch=9/80 step=20/399840 lr=0.00500 loss=6.0172 step/sec=4.674 | ETA 23:45:49 Save model checkpoint to snapshots/glore_res101_pascalContext/ epoch=15/80 step=30/399840 lr=0.00500 loss=5.4674 step/sec=3.519 | ETA 31:33:22 Save model checkpoint to snapshots/glore_res101_pascalContext/ epoch=22/80 step=40/399840 lr=0.00500 loss=5.5174 step/sec=3.479 | ETA 31:55:30 epoch=26/80 step=50/399840 lr=0.00500 loss=5.5387 step/sec=4.506 | ETA 24:38:50 Save model checkpoint to snapshots/glore_res101_pascalContext/ epoch=31/80 step=60/399840 lr=0.00500 loss=5.6265 step/sec=3.497 | ETA 31:45:20 epoch=35/80 step=70/399840 lr=0.00500 loss=5.3708 step/sec=4.590 | ETA 24:11:37 epoch=38/80 step=80/399840 lr=0.00500 loss=5.2788 step/sec=4.638 | ETA 23:56:38 Save model checkpoint to snapshots/glore_res101_pascalContext/ epoch=42/80 step=90/399840 lr=0.00500 loss=5.7635 step/sec=3.593 | ETA 30:54:12 epoch=46/80 step=100/399840 lr=0.00500 loss=4.9700 step/sec=4.555 | ETA 24:22:45 Save model checkpoint to snapshots/glore_res101_pascalContext/ epoch=51/80 step=110/399840 lr=0.00500 loss=5.2388 step/sec=3.539 | ETA 31:22:37 epoch=56/80 step=120/399840 lr=0.00500 loss=5.0396 step/sec=4.667 | ETA 23:47:30 Save model checkpoint to snapshots/glore_res101_pascalContext/ epoch=66/80 step=130/399840 lr=0.00500 loss=5.0537 step/sec=3.484 | ETA 31:52:21 epoch=67/80 step=140/399840 lr=0.00500 loss=4.4081 step/sec=4.787 | ETA 23:11:34 Save model checkpoint to snapshots/glore_res101_pascalContext/ epoch=76/80 step=150/399840 lr=0.00500 loss=5.2291 step/sec=3.409 | ETA 32:34:16 epoch=77/80 step=160/399840 lr=0.00500 loss=5.3577 step/sec=4.826 | ETA 23:00:22 epoch=78/80 step=170/399840 lr=0.00500 loss=5.2874 step/sec=4.823 | ETA 23:01:04 Save model checkpoint to snapshots/glore_res101_pascalContext/ Save model checkpoint to snapshots/glore_res101_pascalContext/ {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.005, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_pascalContext/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 1, 'TRAIN_CROP_SIZE': (520, 520)} #device count: 1 #train_batch_size: 1 #batch_size_per_dev: 1 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 4225) logits.type: Sync BatchNorm strategy will not be effective if GPU device count <= 1 Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader epoch=1/80 step=10/399840 lr=0.00500 loss=6.5236 step/sec=3.092 | ETA 35:54:52 epoch=1/80 step=20/399840 lr=0.00500 loss=5.9645 step/sec=4.837 | ETA 22:57:30 epoch=1/80 step=30/399840 lr=0.00500 loss=5.8043 step/sec=4.849 | ETA 22:54:14 epoch=1/80 step=40/399840 lr=0.00500 loss=5.7938 step/sec=4.857 | ETA 22:51:46 epoch=1/80 step=50/399840 lr=0.00500 loss=5.5355 step/sec=4.854 | ETA 22:52:38 epoch=1/80 step=60/399840 lr=0.00500 loss=5.5231 step/sec=4.871 | ETA 22:47:49 epoch=1/80 step=70/399840 lr=0.00500 loss=5.1272 step/sec=4.877 | ETA 22:46:03 epoch=1/80 step=80/399840 lr=0.00500 loss=5.2581 step/sec=4.882 | ETA 22:44:38 epoch=1/80 step=90/399840 lr=0.00500 loss=4.8484 step/sec=4.884 | ETA 22:44:17 epoch=1/80 step=100/399840 lr=0.00500 loss=5.2630 step/sec=4.880 | ETA 22:45:13 epoch=1/80 step=110/399840 lr=0.00500 loss=4.4485 step/sec=4.882 | ETA 22:44:43 epoch=1/80 step=120/399840 lr=0.00500 loss=5.4158 step/sec=4.889 | ETA 22:42:34 epoch=1/80 step=130/399840 lr=0.00500 loss=5.2440 step/sec=4.885 | ETA 22:43:50 epoch=1/80 step=140/399840 lr=0.00500 loss=4.6480 step/sec=4.887 | ETA 22:43:06 epoch=1/80 step=150/399840 lr=0.00500 loss=5.4651 step/sec=4.883 | ETA 22:44:10 epoch=1/80 step=160/399840 lr=0.00500 loss=5.3663 step/sec=4.887 | ETA 22:43:09 epoch=1/80 step=170/399840 lr=0.00500 loss=4.8083 step/sec=4.882 | ETA 22:44:34 epoch=1/80 step=180/399840 lr=0.00500 loss=4.8164 step/sec=4.881 | ETA 22:44:37 epoch=1/80 step=190/399840 lr=0.00500 loss=4.8652 step/sec=4.874 | ETA 22:46:29 epoch=1/80 step=200/399840 lr=0.00500 loss=5.2181 step/sec=4.885 | ETA 22:43:27 epoch=1/80 step=210/399840 lr=0.00500 loss=4.5346 step/sec=4.888 | ETA 22:42:33 epoch=1/80 step=220/399840 lr=0.00500 loss=4.4484 step/sec=4.884 | ETA 22:43:49 epoch=1/80 step=230/399840 lr=0.00500 loss=5.2132 step/sec=4.877 | ETA 22:45:39 epoch=1/80 step=240/399840 lr=0.00500 loss=4.9915 step/sec=4.885 | ETA 22:43:19 epoch=1/80 step=250/399840 lr=0.00500 loss=4.6150 step/sec=4.872 | ETA 22:46:55 epoch=1/80 step=260/399840 lr=0.00500 loss=4.8601 step/sec=4.889 | ETA 22:42:08 epoch=1/80 step=270/399840 lr=0.00500 loss=4.8813 step/sec=4.881 | ETA 22:44:18 epoch=1/80 step=280/399840 lr=0.00500 loss=4.3682 step/sec=4.885 | ETA 22:43:09 epoch=1/80 step=290/399840 lr=0.00500 loss=4.5786 step/sec=4.883 | ETA 22:43:49 epoch=1/80 step=300/399840 lr=0.00500 loss=5.0191 step/sec=4.877 | ETA 22:45:19 epoch=1/80 step=310/399840 lr=0.00500 loss=5.0581 step/sec=4.878 | ETA 22:45:11 epoch=1/80 step=320/399840 lr=0.00500 loss=5.1431 step/sec=4.882 | ETA 22:43:53 epoch=1/80 step=330/399840 lr=0.00500 loss=4.5379 step/sec=4.881 | ETA 22:44:10 epoch=1/80 step=340/399840 lr=0.00500 loss=4.7337 step/sec=4.892 | ETA 22:41:06 epoch=1/80 step=350/399840 lr=0.00500 loss=4.6240 step/sec=4.889 | ETA 22:41:53 epoch=1/80 step=360/399840 lr=0.00500 loss=4.5021 step/sec=4.892 | ETA 22:40:51 epoch=1/80 step=370/399840 lr=0.00500 loss=5.0534 step/sec=4.882 | ETA 22:43:52 epoch=1/80 step=380/399840 lr=0.00500 loss=5.3639 step/sec=4.895 | ETA 22:40:06 epoch=1/80 step=390/399840 lr=0.00500 loss=4.8748 step/sec=4.893 | ETA 22:40:37 epoch=1/80 step=400/399840 lr=0.00500 loss=5.0005 step/sec=4.880 | ETA 22:44:15 epoch=1/80 step=410/399840 lr=0.00500 loss=4.8577 step/sec=4.892 | ETA 22:40:43 epoch=1/80 step=420/399840 lr=0.00500 loss=4.9999 step/sec=4.888 | ETA 22:41:56 epoch=1/80 step=430/399840 lr=0.00500 loss=4.6285 step/sec=4.881 | ETA 22:43:52 epoch=1/80 step=440/399840 lr=0.00500 loss=4.8234 step/sec=4.890 | ETA 22:41:20 epoch=1/80 step=450/399840 lr=0.00499 loss=4.4848 step/sec=4.895 | ETA 22:39:53 epoch=1/80 step=460/399840 lr=0.00499 loss=4.4665 step/sec=4.898 | ETA 22:39:04 epoch=1/80 step=470/399840 lr=0.00499 loss=5.1641 step/sec=4.894 | ETA 22:39:58 epoch=1/80 step=480/399840 lr=0.00499 loss=4.6689 step/sec=4.889 | ETA 22:41:18 epoch=1/80 step=490/399840 lr=0.00499 loss=4.1085 step/sec=4.895 | ETA 22:39:44 epoch=1/80 step=500/399840 lr=0.00499 loss=4.6647 step/sec=4.901 | ETA 22:38:03 epoch=1/80 step=510/399840 lr=0.00499 loss=4.5266 step/sec=4.899 | ETA 22:38:39 epoch=1/80 step=520/399840 lr=0.00499 loss=4.4778 step/sec=4.893 | ETA 22:40:02 epoch=1/80 step=530/399840 lr=0.00499 loss=5.1426 step/sec=4.904 | ETA 22:37:06 epoch=1/80 step=540/399840 lr=0.00499 loss=4.5781 step/sec=4.886 | ETA 22:42:01 epoch=1/80 step=550/399840 lr=0.00499 loss=5.6345 step/sec=4.891 | ETA 22:40:35 epoch=1/80 step=560/399840 lr=0.00499 loss=4.7484 step/sec=4.898 | ETA 22:38:36 epoch=1/80 step=570/399840 lr=0.00499 loss=4.3737 step/sec=4.892 | ETA 22:40:21 epoch=1/80 step=580/399840 lr=0.00499 loss=4.8498 step/sec=4.903 | ETA 22:37:16 epoch=1/80 step=590/399840 lr=0.00499 loss=4.4792 step/sec=4.891 | ETA 22:40:32 epoch=1/80 step=600/399840 lr=0.00499 loss=4.0987 step/sec=4.893 | ETA 22:40:00 epoch=1/80 step=610/399840 lr=0.00499 loss=4.2048 step/sec=4.895 | ETA 22:39:12 epoch=1/80 step=620/399840 lr=0.00499 loss=4.8651 step/sec=4.896 | ETA 22:38:57 epoch=1/80 step=630/399840 lr=0.00499 loss=3.9858 step/sec=4.899 | ETA 22:38:08 epoch=1/80 step=640/399840 lr=0.00499 loss=4.2324 step/sec=4.891 | ETA 22:40:18 epoch=1/80 step=650/399840 lr=0.00499 loss=4.3032 step/sec=4.899 | ETA 22:37:57 epoch=1/80 step=660/399840 lr=0.00499 loss=4.4643 step/sec=4.887 | ETA 22:41:20 epoch=1/80 step=670/399840 lr=0.00499 loss=4.2801 step/sec=4.883 | ETA 22:42:19 epoch=1/80 step=680/399840 lr=0.00499 loss=4.7291 step/sec=4.889 | ETA 22:40:49 epoch=1/80 step=690/399840 lr=0.00499 loss=4.3516 step/sec=4.888 | ETA 22:41:06 epoch=1/80 step=700/399840 lr=0.00499 loss=4.1758 step/sec=4.887 | ETA 22:41:22 epoch=1/80 step=710/399840 lr=0.00499 loss=4.7739 step/sec=4.891 | ETA 22:40:09 epoch=1/80 step=720/399840 lr=0.00499 loss=4.4306 step/sec=4.878 | ETA 22:43:39 epoch=1/80 step=730/399840 lr=0.00499 loss=4.8173 step/sec=4.891 | ETA 22:40:05 epoch=1/80 step=740/399840 lr=0.00499 loss=3.7910 step/sec=4.892 | ETA 22:39:42 epoch=1/80 step=750/399840 lr=0.00499 loss=4.7324 step/sec=4.892 | ETA 22:39:35 epoch=1/80 step=760/399840 lr=0.00499 loss=4.7868 step/sec=4.886 | ETA 22:41:10 epoch=1/80 step=770/399840 lr=0.00499 loss=4.8752 step/sec=4.894 | ETA 22:39:05 epoch=1/80 step=780/399840 lr=0.00499 loss=4.8768 step/sec=4.891 | ETA 22:39:52 epoch=1/80 step=790/399840 lr=0.00499 loss=4.1491 step/sec=4.892 | ETA 22:39:27 epoch=1/80 step=800/399840 lr=0.00499 loss=4.3457 step/sec=4.888 | ETA 22:40:35 epoch=1/80 step=810/399840 lr=0.00499 loss=4.4186 step/sec=4.892 | ETA 22:39:19 epoch=1/80 step=820/399840 lr=0.00499 loss=4.3745 step/sec=4.893 | ETA 22:39:07 epoch=1/80 step=830/399840 lr=0.00499 loss=4.0051 step/sec=4.889 | ETA 22:40:12 epoch=1/80 step=840/399840 lr=0.00499 loss=4.4603 step/sec=4.893 | ETA 22:39:03 epoch=1/80 step=850/399840 lr=0.00499 loss=5.4198 step/sec=4.889 | ETA 22:40:17 epoch=1/80 step=860/399840 lr=0.00499 loss=4.2998 step/sec=4.886 | ETA 22:40:49 epoch=1/80 step=870/399840 lr=0.00499 loss=4.1961 step/sec=4.895 | ETA 22:38:27 epoch=1/80 step=880/399840 lr=0.00499 loss=5.0197 step/sec=4.889 | ETA 22:40:05 epoch=1/80 step=890/399840 lr=0.00499 loss=4.3922 step/sec=4.880 | ETA 22:42:32 epoch=1/80 step=900/399840 lr=0.00499 loss=4.3486 step/sec=4.892 | ETA 22:39:09 epoch=1/80 step=910/399840 lr=0.00499 loss=4.1876 step/sec=4.888 | ETA 22:40:14 epoch=1/80 step=920/399840 lr=0.00499 loss=3.9571 step/sec=4.893 | ETA 22:38:42 epoch=1/80 step=930/399840 lr=0.00499 loss=4.5653 step/sec=4.881 | ETA 22:42:01 epoch=1/80 step=940/399840 lr=0.00499 loss=4.6662 step/sec=4.883 | ETA 22:41:23 epoch=1/80 step=950/399840 lr=0.00499 loss=4.4738 step/sec=4.887 | ETA 22:40:30 epoch=1/80 step=960/399840 lr=0.00499 loss=4.5106 step/sec=4.889 | ETA 22:39:51 epoch=1/80 step=970/399840 lr=0.00499 loss=5.1103 step/sec=4.886 | ETA 22:40:38 epoch=1/80 step=980/399840 lr=0.00499 loss=4.1730 step/sec=4.901 | ETA 22:36:29 epoch=1/80 step=990/399840 lr=0.00499 loss=4.3621 step/sec=4.892 | ETA 22:38:45 epoch=1/80 step=1000/399840 lr=0.00499 loss=4.4590 step/sec=4.884 | ETA 22:40:59 {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.005, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_pascalContext/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 1, 'TRAIN_CROP_SIZE': (520, 520)} #device count: 1 #train_batch_size: 1 #batch_size_per_dev: 1 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 4225) Sync BatchNorm strategy will not be effective if GPU device count <= 1 Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader epoch=1/80 step=10/399840 lr=0.00500 loss=4.6802 step/sec=3.544 | ETA 31:20:09 epoch=1/80 step=20/399840 lr=0.00500 loss=3.9633 step/sec=5.721 | ETA 19:24:41 epoch=1/80 step=30/399840 lr=0.00500 loss=4.1571 step/sec=5.731 | ETA 19:22:42 epoch=1/80 step=40/399840 lr=0.00500 loss=3.7191 step/sec=5.728 | ETA 19:23:13 epoch=1/80 step=50/399840 lr=0.00500 loss=4.2360 step/sec=5.720 | ETA 19:24:54 epoch=1/80 step=60/399840 lr=0.00500 loss=4.1186 step/sec=5.732 | ETA 19:22:24 epoch=1/80 step=70/399840 lr=0.00500 loss=4.2668 step/sec=5.723 | ETA 19:24:09 epoch=1/80 step=80/399840 lr=0.00500 loss=3.7505 step/sec=5.735 | ETA 19:21:46 epoch=1/80 step=90/399840 lr=0.00500 loss=3.6923 step/sec=5.731 | ETA 19:22:26 epoch=1/80 step=100/399840 lr=0.00500 loss=3.9183 step/sec=5.741 | ETA 19:20:26 epoch=1/80 step=110/399840 lr=0.00500 loss=3.6083 step/sec=5.717 | ETA 19:25:17 epoch=1/80 step=120/399840 lr=0.00500 loss=3.3056 step/sec=5.740 | ETA 19:20:40 epoch=1/80 step=130/399840 lr=0.00500 loss=3.4347 step/sec=5.741 | ETA 19:20:27 epoch=1/80 step=140/399840 lr=0.00500 loss=4.4239 step/sec=5.737 | ETA 19:21:07 epoch=1/80 step=150/399840 lr=0.00500 loss=3.5880 step/sec=5.724 | ETA 19:23:52 epoch=1/80 step=160/399840 lr=0.00500 loss=3.6183 step/sec=5.717 | ETA 19:25:10 epoch=1/80 step=170/399840 lr=0.00500 loss=3.3986 step/sec=5.721 | ETA 19:24:16 epoch=1/80 step=180/399840 lr=0.00500 loss=3.3740 step/sec=5.722 | ETA 19:24:04 epoch=1/80 step=190/399840 lr=0.00500 loss=3.7950 step/sec=5.715 | ETA 19:25:23 epoch=1/80 step=200/399840 lr=0.00500 loss=3.8290 step/sec=5.716 | ETA 19:25:21 epoch=1/80 step=210/399840 lr=0.00500 loss=3.1311 step/sec=5.727 | ETA 19:23:03 {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.005, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_pascalContext/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4, 'TRAIN_CROP_SIZE': (520, 520)} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 4225) Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader epoch=1/80 step=10/49920 lr=0.00500 loss=4.4876 step/sec=0.388 | ETA 35:41:37 epoch=1/80 step=20/49920 lr=0.00500 loss=4.1046 step/sec=0.555 | ETA 24:59:19 epoch=1/80 step=30/49920 lr=0.00500 loss=3.6741 step/sec=0.555 | ETA 24:57:28 epoch=1/80 step=40/49920 lr=0.00500 loss=3.6090 step/sec=0.555 | ETA 24:57:08 epoch=1/80 step=50/49920 lr=0.00500 loss=3.6953 step/sec=0.554 | ETA 25:00:08 epoch=1/80 step=60/49920 lr=0.00499 loss=3.5173 step/sec=0.552 | ETA 25:05:20 epoch=1/80 step=70/49920 lr=0.00499 loss=3.3773 step/sec=0.553 | ETA 25:01:34 epoch=1/80 step=80/49920 lr=0.00499 loss=3.3467 step/sec=0.553 | ETA 25:02:37 epoch=1/80 step=90/49920 lr=0.00499 loss=3.4842 step/sec=0.555 | ETA 24:56:33 {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'ADEChallengeData2016', 'DATA_DIM': 3, 'DATA_DIR': './data_local/ADEChallengeData2016/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 150, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'TEST_TOTAL_IMAGES': 2000, 'TRAIN_FILE_LIST': './data_local/ADEChallengeData2016/ade_train.lst', 'TRAIN_TOTAL_IMAGES': 20210, 'VAL_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'VAL_TOTAL_IMAGES': 2000, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 120, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_ade/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_ade/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4, 'TRAIN_CROP_SIZE': (520, 520)} {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'ADEChallengeData2016', 'DATA_DIM': 3, 'DATA_DIR': './data_local/ADEChallengeData2016/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 150, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'TEST_TOTAL_IMAGES': 2000, 'TRAIN_FILE_LIST': './data_local/ADEChallengeData2016/ade_train.lst', 'TRAIN_TOTAL_IMAGES': 20210, 'VAL_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'VAL_TOTAL_IMAGES': 2000, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 120, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_ade/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_ade/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4, 'TRAIN_CROP_SIZE': (520, 520)} {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'ADEChallengeData2016', 'DATA_DIM': 3, 'DATA_DIR': './data_local/ADEChallengeData2016/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 150, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'TEST_TOTAL_IMAGES': 2000, 'TRAIN_FILE_LIST': './data_local/ADEChallengeData2016/ade_train.lst', 'TRAIN_TOTAL_IMAGES': 20210, 'VAL_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'VAL_TOTAL_IMAGES': 2000, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 120, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_ade/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_ade/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4, 'TRAIN_CROP_SIZE': (520, 520)} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 4225) Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'ADEChallengeData2016', 'DATA_DIM': 3, 'DATA_DIR': './data_local/ADEChallengeData2016/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 150, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'TEST_TOTAL_IMAGES': 2000, 'TRAIN_FILE_LIST': './data_local/ADEChallengeData2016/ade_train.lst', 'TRAIN_TOTAL_IMAGES': 20210, 'VAL_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'VAL_TOTAL_IMAGES': 2000, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 120, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_ade/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_ade/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4, 'TRAIN_CROP_SIZE': (520, 520)} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 4225) Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'ADEChallengeData2016', 'DATA_DIM': 3, 'DATA_DIR': './data_local/ADEChallengeData2016/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 150, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\\t', 'TEST_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'TEST_TOTAL_IMAGES': 2000, 'TRAIN_FILE_LIST': './data_local/ADEChallengeData2016/ade_train.lst', 'TRAIN_TOTAL_IMAGES': 20210, 'VAL_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'VAL_TOTAL_IMAGES': 2000, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 120, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_ade/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_ade/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4, 'TRAIN_CROP_SIZE': (520, 520)} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 4225) Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'ADEChallengeData2016', 'DATA_DIM': 3, 'DATA_DIR': './data_local/ADEChallengeData2016/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 150, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'TEST_TOTAL_IMAGES': 2000, 'TRAIN_FILE_LIST': './data_local/ADEChallengeData2016/ade_train.lst', 'TRAIN_TOTAL_IMAGES': 20210, 'VAL_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'VAL_TOTAL_IMAGES': 2000, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 120, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_ade/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_ade/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4, 'TRAIN_CROP_SIZE': (520, 520)} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 4225) Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'ADEChallengeData2016', 'DATA_DIM': 3, 'DATA_DIR': './data_local/ADEChallengeData2016/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 150, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\\t', 'TEST_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'TEST_TOTAL_IMAGES': 2000, 'TRAIN_FILE_LIST': './data_local/ADEChallengeData2016/ade_train.lst', 'TRAIN_TOTAL_IMAGES': 20210, 'VAL_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'VAL_TOTAL_IMAGES': 2000, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 120, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_ade/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_ade/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4, 'TRAIN_CROP_SIZE': (520, 520)} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 4225) Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ Save model checkpoint to snapshots/glore_res101_ade/ {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'ADEChallengeData2016', 'DATA_DIM': 3, 'DATA_DIR': './data_local/ADEChallengeData2016/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 150, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'TEST_TOTAL_IMAGES': 2000, 'TRAIN_FILE_LIST': './data_local/ADEChallengeData2016/ade_train.lst', 'TRAIN_TOTAL_IMAGES': 20210, 'VAL_FILE_LIST': './data_local/ADEChallengeData2016/ade_val.lst', 'VAL_TOTAL_IMAGES': 2000, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'EVAL_CROP_SIZE': (520, 520), 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': False, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 120, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'TEST_MODEL': 'snapshots/glore_res101_ade/final'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_ade/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4, 'TRAIN_CROP_SIZE': (520, 520)} #device count: 1 #train_batch_size: 4 #batch_size_per_dev: 4 func_name: modeling.glore.glore module_name: src.models.modeling.glore (-1, 128, 64) aaa (-1, 64, 128) (-1, 64, 128) (-1, 64, 4225) Sync BatchNorm strategy will not be effective if GPU device count <= 1 Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader epoch=1/120 step=10/606240 lr=0.01000 loss=5.2836 step/sec=1.161 | ETA 145:03:30 epoch=1/120 step=20/606240 lr=0.01000 loss=4.2554 step/sec=1.329 | ETA 126:41:09 epoch=1/120 step=30/606240 lr=0.01000 loss=3.9273 step/sec=1.333 | ETA 126:18:14 {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 198, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.005, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader epoch=1/80 step=10/1920 lr=0.00498 loss=6.2260 step/sec=0.577 | ETA 00:55:07 epoch=1/80 step=20/1920 lr=0.00496 loss=5.7186 step/sec=1.000 | ETA 00:31:39 epoch=2/80 step=30/1920 lr=0.00493 loss=5.2041 step/sec=0.979 | ETA 00:32:10 {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 198, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.005, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 1, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 10, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader epoch=1/1 step=10/24 lr=0.00341 loss=6.1900 step/sec=0.591 | ETA 00:00:23 epoch=1/1 step=20/24 lr=0.00153 loss=5.7847 step/sec=1.027 | ETA 00:00:03 Save model checkpoint to snapshots/glore_res101_pascalContext/ {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 198, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 8, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.005, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 2, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model_local/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model_local/resnet101_v2/ not exists, training from scratch... Use multi-thread reader epoch=1/2 step=10/48 lr=0.00420 loss=6.2596 step/sec=0.437 | ETA 00:01:26 epoch=1/2 step=20/48 lr=0.00329 loss=5.6335 step/sec=0.653 | ETA 00:00:42 Save model checkpoint to snapshots/glore_res101_pascalContext/ epoch=2/2 step=30/48 lr=0.00225 loss=5.2856 step/sec=0.625 | ETA 00:00:28 epoch=2/2 step=40/48 lr=0.00126 loss=5.1502 step/sec=0.648 | ETA 00:00:12 Save model checkpoint to snapshots/glore_res101_pascalContext/ Save model checkpoint to snapshots/glore_res101_pascalContext/ {'DATAAUG': {'BASE_SIZE': 1024, 'CROP_SIZE': 769, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.75}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'cityscapes', 'DATA_DIM': 3, 'DATA_DIR': './data_local/cityscapes/', 'IGNORE_INDEX': 255, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 19, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': '\t', 'TEST_FILE_LIST': './data_local/cityscapes/cityscapes_list/test.lst', 'TEST_TOTAL_IMAGES': 1525, 'TRAIN_FILE_LIST': './data_local/cityscapes/cityscapes_list/train.lst', 'TRAIN_TOTAL_IMAGES': 2975, 'VAL_FILE_LIST': './data_local/cityscapes/cityscapes_list/val.lst', 'VAL_TOTAL_IMAGES': 500, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 1, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'deeplabv3', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 80, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 2048, 'CROP_SIZE': 769, 'SLIDE_WINDOW': True, 'TEST_MODEL': './snapshots/deeplabv3_resnet_cityscapes'}, 'TRAIN': {'MODEL_SAVE_DIR': './snapshots/deeplabv3_resnet_cityscapes/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 2} #device count: 2 #train_batch_size: 4 #batch_size_per_dev: 2 iterable: False func_name: modeling.deeplabv3.deeplabv3 module_name: src.models.modeling.deeplabv3 logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/80 step=10/59440 lr=0.01000 loss=3.4193 step/sec=0.359 | ETA 45:56:52 epoch=1/80 step=20/59440 lr=0.01000 loss=2.6701 step/sec=0.536 | ETA 30:46:24 epoch=1/80 step=30/59440 lr=0.01000 loss=2.7055 step/sec=0.533 | ETA 30:56:29 {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 1, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_MULTI_GRID': True, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 90, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 1, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_MULTI_GRID': True, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 90, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore employ multi-grid for resnet backbone network employ multi-grid for resnet backbone network employ multi-grid for resnet backbone network (-1, 128, 64) res5.shape: (-1, 2048, 65, 65) res4.shape: (-1, 1024, 65, 65) logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error] -------------------------------------------- C++ Call Stacks (More useful to developers): -------------------------------------------- 0 std::string paddle::platform::GetTraceBackString(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::get_mid_dims(paddle::framework::DDim const&, paddle::framework::DDim const&, int, int*, int*, int*, int*) 3 void paddle::operators::ElementwiseComputeEx, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::AddFunctor, paddle::framework::Tensor*) 4 void paddle::operators::default_elementwise_add(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) 5 paddle::operators::ElementwiseAddKernel::Compute(paddle::framework::ExecutionContext const&) const 6 std::_Function_handler, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel, paddle::operators::ElementwiseAddKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const 8 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const 9 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) 10 paddle::framework::details::ComputationOpHandle::RunImpl() 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 12 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr > const&, unsigned long*) 13 std::_Function_handler (), std::__future_base::_Task_setter, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 14 std::__future_base::_State_base::_M_do_set(std::function ()>&, bool&) 15 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const ------------------------------------------ Python Call Stacks (More useful to users): ------------------------------------------ File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2459, in append_op attrs=kwargs.get("attrs", None)) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 13977, in _elementwise_op 'use_mkldnn': use_mkldnn}) File "/home/wutianyi/anaconda3/envs/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14119, in elementwise_add return _elementwise_op(LayerHelper('elementwise_add', **locals())) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 257, in bottleneck_block x=short, y=conv2, act='relu', name=name + ".add.output.5") File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/backbone/resnet.py", line 118, in net dilation=dilation_rate) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 93, in resnet decode_points= decode_points) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/modeling/glore.py", line 104, in glore res5, feat_dict = resnet(input) File "/ssd1/wutianyi0927/dlcvcluster/semseg_paddle_c_v1/src/models/model_builder.py", line 178, in build_model logits = model_func(image, class_num) File "train.py", line 187, in train train_prog, startup_prog, phase=ModelPhase.TRAIN) File "train.py", line 422, in main train(cfg) File "train.py", line 432, in main(args) ---------------------- Error Message Summary: ---------------------- PaddleCheckError: Expected y_dims[i] == 1, but received y_dims[i]:69 != 1:1. ShapeError: broadcast dimension mismatch. Operands could not be broadcast together with the shape of X = [4, 1024, 65, 65] and the shape of Y = [4, 1024, 69, 69]. Received [65] in X is not equal to [69] in Y at [/paddle/paddle/fluid/operators/elementwise/elementwise_op_function.h:79] [operator < elementwise_add > error]{'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 1, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_MULTI_GRID': True, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 90, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore employ multi-grid for resnet backbone network employ multi-grid for resnet backbone network employ multi-grid for resnet backbone network (-1, 128, 64) res5.shape: (-1, 2048, 65, 65) res4.shape: (-1, 1024, 65, 65) logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 1, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_MULTI_GRID': True, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 90, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore employ multi-grid for resnet backbone network employ multi-grid for resnet backbone network employ multi-grid for resnet backbone network (-1, 128, 64) res5.shape: (-1, 2048, 65, 65) res4.shape: (-1, 1024, 65, 65) logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/90 step=10/56160 lr=0.01000 loss=6.0952 step/sec=0.311 | ETA 50:08:45 epoch=1/90 step=20/56160 lr=0.01000 loss=5.6416 step/sec=0.403 | ETA 38:42:27 epoch=1/90 step=30/56160 lr=0.01000 loss=5.2850 step/sec=0.403 | ETA 38:43:54 {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 1, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_MULTI_GRID': True, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 90, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore employ multi-grid for resnet backbone network employ multi-grid for resnet backbone network employ multi-grid for resnet backbone network (-1, 128, 64) res5.shape: (-1, 2048, 65, 65) res4.shape: (-1, 1024, 65, 65) logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 1, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_MULTI_GRID': True, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 90, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore employ multi-grid for resnet backbone network employ multi-grid for resnet backbone network employ multi-grid for resnet backbone network (-1, 128, 64) res5.shape: (-1, 2048, 65, 65) res4.shape: (-1, 1024, 65, 65) logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/90 step=10/56160 lr=0.01000 loss=6.1452 step/sec=0.412 | ETA 37:48:42 epoch=1/90 step=20/56160 lr=0.01000 loss=5.5765 step/sec=0.650 | ETA 23:59:54 {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 1, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_MULTI_GRID': True, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 90, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore employ multi-grid for resnet backbone network employ multi-grid for resnet backbone network employ multi-grid for resnet backbone network (-1, 128, 64) res5.shape: (-1, 2048, 65, 65) res4.shape: (-1, 1024, 65, 65) logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 1, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_MULTI_GRID': True, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 90, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore employ multi-grid for resnet backbone network: dilation_rate={} employ multi-grid for resnet backbone network: dilation_rate={} employ multi-grid for resnet backbone network: dilation_rate={} (-1, 128, 64) res5.shape: (-1, 2048, 65, 65) res4.shape: (-1, 1024, 65, 65) logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/90 step=10/56160 lr=0.01000 loss=6.0117 step/sec=0.416 | ETA 37:31:58 {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 1, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_MULTI_GRID': True, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 90, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore employ multi-grid for resnet backbone network: dilation_rate=4 employ multi-grid for resnet backbone network: dilation_rate=8 employ multi-grid for resnet backbone network: dilation_rate=16 (-1, 128, 64) res5.shape: (-1, 2048, 65, 65) res4.shape: (-1, 1024, 65, 65) logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader {'DATAAUG': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'EXTRA': True, 'RAND_SCALE_MAX': 2.0, 'RAND_SCALE_MIN': 0.5}, 'DATALOADER': {'BUF_SIZE': 256, 'NUM_WORKERS': 8}, 'DATASET': {'DATASET_NAME': 'pascalContext', 'DATA_DIM': 3, 'DATA_DIR': './data_local/pascalContext/', 'IGNORE_INDEX': -1, 'IMAGE_TYPE': 'rgb', 'NUM_CLASSES': 59, 'PADDING_VALUE': [123.675, 116.28, 103.53], 'SEPARATOR': ' ', 'TEST_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'TEST_TOTAL_IMAGES': 5103, 'TRAIN_FILE_LIST': './data_local/pascalContext/pascal_context_train.txt', 'TRAIN_TOTAL_IMAGES': 4998, 'VAL_FILE_LIST': './data_local/pascalContext/pascal_context_val.txt', 'VAL_TOTAL_IMAGES': 5103, 'VIS_FILE_LIST': None}, 'EVAL_BATCH_SIZE': 1, 'MEAN': [0.485, 0.456, 0.406], 'MODEL': {'BACKBONE': 'resnet', 'BACKBONE_LAYERS': 101, 'BACKBONE_MULTI_GRID': True, 'BACKBONE_OUTPUT_STRIDE': 8, 'BN_MOMENTUM': 0.99, 'DEEPLABv3': {'ASPP_WITH_SEP_CONV': True, 'AuxHead': True, 'DEPTH_MULTIPLIER': 1.0}, 'DEFAULT_EPSILON': 1e-05, 'DEFAULT_GROUP_NUMBER': 32, 'DEFAULT_NORM_TYPE': 'bn', 'FP16': False, 'GLORE': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'HRNET': {'STAGE2': {'NUM_CHANNELS': [40, 80], 'NUM_MODULES': 1}, 'STAGE3': {'NUM_CHANNELS': [40, 80, 160], 'NUM_MODULES': 4}, 'STAGE4': {'NUM_CHANNELS': [40, 80, 160, 320], 'NUM_MODULES': 3}}, 'MODEL_NAME': 'glore', 'MULTI_LOSS_WEIGHT': [1.0, 0.4], 'PSPNET': {'AuxHead': True, 'DEPTH_MULTIPLIER': 1}, 'SCALE_LOSS': 'DYNAMIC'}, 'NUM_TRAINERS': 1, 'SOLVER': {'BEGIN_EPOCH': 1, 'DECAY_EPOCH': [10, 20], 'GAMMA': 0.1, 'LOSS': ['softmax_loss'], 'LR': 0.01, 'LR_POLICY': 'poly', 'LR_WARMUP': False, 'LR_WARMUP_STEPS': 2000, 'MOMENTUM': 0.9, 'MOMENTUM2': 0.999, 'NUM_EPOCHS': 90, 'OPTIMIZER': 'sgd', 'POWER': 0.9, 'WEIGHT_DECAY': 4e-05}, 'STD': [0.229, 0.224, 0.225], 'TEST': {'BASE_SIZE': 520, 'CROP_SIZE': 520, 'SLIDE_WINDOW': True, 'TEST_MODEL': 'snapshots/glore_res101_pascalContext'}, 'TRAIN': {'MODEL_SAVE_DIR': 'snapshots/glore_res101_pascalContext/', 'PRETRAINED_MODEL_DIR': './pretrained_model/resnet101_v2/', 'RESUME_MODEL_DIR': '', 'SNAPSHOT_EPOCH': 1, 'SYNC_BATCH_NORM': True}, 'TRAINER_ID': 0, 'TRAIN_BATCH_SIZE': 8, 'TRAIN_BATCH_SIZE_PER_GPU': 4} #device count: 2 #train_batch_size: 8 #batch_size_per_dev: 4 iterable: False func_name: modeling.glore.glore module_name: src.models.modeling.glore employ multi-grid for resnet backbone network: dilation_rate=4 employ multi-grid for resnet backbone network: dilation_rate=8 employ multi-grid for resnet backbone network: dilation_rate=16 (-1, 128, 64) res5.shape: (-1, 2048, 65, 65) res4.shape: (-1, 1024, 65, 65) logits.type: Sync BatchNorm strategy is effective. Pretrained model dir ./pretrained_model/resnet101_v2/ not exists, training from scratch... Use multiprocess reader epoch=1/90 step=10/56160 lr=0.01000 loss=6.2111 step/sec=0.410 | ETA 38:03:15 epoch=1/90 step=20/56160 lr=0.01000 loss=5.6087 step/sec=0.645 | ETA 24:11:31