diff --git a/configs/dcn/yolov3_enhance_reader.yml b/configs/dcn/yolov3_enhance_reader.yml
index 0cb379d2542587d04fee3cd68d42adf22ec18031..db72a924b4a237d27150baa8a70382543d7f1f20 100644
--- a/configs/dcn/yolov3_enhance_reader.yml
+++ b/configs/dcn/yolov3_enhance_reader.yml
@@ -2,7 +2,6 @@ TrainReader:
   inputs_def:
     fields: ['image', 'gt_bbox', 'gt_class', 'gt_score']
     num_max_boxes: 50
-  use_fine_grained_loss: true
   dataset:
     !COCODataSet
     image_dir: train2017
diff --git a/configs/dcn/yolov3_r50vd_dcn_db_iouaware_obj365_pretrained_coco.yml b/configs/dcn/yolov3_r50vd_dcn_db_iouaware_obj365_pretrained_coco.yml
index 6177aaac7f26dd0c6395a040cddfa11ffc705d39..f40622036c40b6b0fc87c199c2c248d20f8914a1 100755
--- a/configs/dcn/yolov3_r50vd_dcn_db_iouaware_obj365_pretrained_coco.yml
+++ b/configs/dcn/yolov3_r50vd_dcn_db_iouaware_obj365_pretrained_coco.yml
@@ -8,12 +8,10 @@ metric: COCO
 pretrain_weights: https://paddlemodels.bj.bcebos.com/object_detection/ResNet50_vd_dcn_db_obj365_pretrained.tar
 weights: output/yolov3_r50vd_dcn_db_iouaware_obj365_pretrained_coco/model_final
 num_classes: 80
-use_fine_grained_loss: true
 
 YOLOv3:
   backbone: ResNet
   yolo_head: YOLOv3Head
-  use_fine_grained_loss: true
 
 ResNet:
   norm_type: sync_bn
@@ -47,7 +45,6 @@ YOLOv3Loss:
   batch_size: 8
   ignore_thresh: 0.7
   label_smooth: false
-  use_fine_grained_loss: true
   iou_loss: IouLoss
   iou_aware_loss: IouAwareLoss
 
diff --git a/configs/dcn/yolov3_r50vd_dcn_db_iouloss_obj365_pretrained_coco.yml b/configs/dcn/yolov3_r50vd_dcn_db_iouloss_obj365_pretrained_coco.yml
index 5e943145330c49b30c1742169a101018187f654c..6f5ec51b7b8ade981223b1b0635917d66277b261 100755
--- a/configs/dcn/yolov3_r50vd_dcn_db_iouloss_obj365_pretrained_coco.yml
+++ b/configs/dcn/yolov3_r50vd_dcn_db_iouloss_obj365_pretrained_coco.yml
@@ -8,12 +8,10 @@ metric: COCO
 pretrain_weights: https://paddlemodels.bj.bcebos.com/object_detection/ResNet50_vd_dcn_db_obj365_pretrained.tar
 weights: output/yolov3_r50vd_dcn_db_iouloss_obj365_pretrained_coco/model_final
 num_classes: 80
-use_fine_grained_loss: true
 
 YOLOv3:
   backbone: ResNet
   yolo_head: YOLOv3Head
-  use_fine_grained_loss: true
 
 ResNet:
   norm_type: sync_bn
@@ -49,7 +47,6 @@ YOLOv3Loss:
   batch_size: 8
   ignore_thresh: 0.7
   label_smooth: false
-  use_fine_grained_loss: true
   iou_loss: IouLoss
 
 IouLoss:
diff --git a/configs/dcn/yolov3_r50vd_dcn_db_obj365_pretrained_coco.yml b/configs/dcn/yolov3_r50vd_dcn_db_obj365_pretrained_coco.yml
index 3c69e410a303212df3539c261465a8cd02e7911c..e0e92d3393c53797b60aa37d35b3dce8e8d2decc 100755
--- a/configs/dcn/yolov3_r50vd_dcn_db_obj365_pretrained_coco.yml
+++ b/configs/dcn/yolov3_r50vd_dcn_db_obj365_pretrained_coco.yml
@@ -8,12 +8,10 @@ metric: COCO
 pretrain_weights: https://paddlemodels.bj.bcebos.com/object_detection/ResNet50_vd_dcn_db_obj365_pretrained.tar
 weights: output/yolov3_r50vd_dcn_db_obj365_pretrained_coco/model_final
 num_classes: 80
-use_fine_grained_loss: true
 
 YOLOv3:
   backbone: ResNet
   yolo_head: YOLOv3Head
-  use_fine_grained_loss: true
 
 ResNet:
   norm_type: sync_bn
@@ -50,7 +48,6 @@ YOLOv3Loss:
   batch_size: 8
   ignore_thresh: 0.7
   label_smooth: false
-  use_fine_grained_loss: true
 
 LearningRate:
   base_lr: 0.001
diff --git a/configs/dcn/yolov3_r50vd_dcn_obj365_pretrained_coco.yml b/configs/dcn/yolov3_r50vd_dcn_obj365_pretrained_coco.yml
index 014a7947e2d711af8d9be1d04139b5058b3b52bf..4a7fe23a6af688abbf89e412c8f92c21722c7716 100755
--- a/configs/dcn/yolov3_r50vd_dcn_obj365_pretrained_coco.yml
+++ b/configs/dcn/yolov3_r50vd_dcn_obj365_pretrained_coco.yml
@@ -8,12 +8,10 @@ metric: COCO
 pretrain_weights: https://paddlemodels.bj.bcebos.com/object_detection/ResNet50_vd_dcn_db_obj365_pretrained.tar
 weights: output/yolov3_r50vd_dcn_db_obj365_pretrained_coco/model_final
 num_classes: 80
-use_fine_grained_loss: true
 
 YOLOv3:
   backbone: ResNet
   yolo_head: YOLOv3Head
-  use_fine_grained_loss: true
 
 ResNet:
   norm_type: sync_bn
@@ -48,7 +46,6 @@ YOLOv3Loss:
   batch_size: 8
   ignore_thresh: 0.7
   label_smooth: false
-  use_fine_grained_loss: true
 
 LearningRate:
   base_lr: 0.001
diff --git a/configs/ppyolo/ppyolo.yml b/configs/ppyolo/ppyolo.yml
index 59d5faa3a45a02cca84d15b48d25cac3d426a425..6c6011d526af2943aa313e30a8211533876d4157 100644
--- a/configs/ppyolo/ppyolo.yml
+++ b/configs/ppyolo/ppyolo.yml
@@ -9,14 +9,12 @@ metric: COCO
 pretrain_weights: https://paddle-imagenet-models-name.bj.bcebos.com/ResNet50_vd_ssld_pretrained.tar
 weights: output/ppyolo/model_final
 num_classes: 80
-use_fine_grained_loss: true
 use_ema: true
 ema_decay: 0.9998
 
 YOLOv3:
   backbone: ResNet
   yolo_head: YOLOv3Head
-  use_fine_grained_loss: true
 
 ResNet:
   norm_type: sync_bn
@@ -48,7 +46,6 @@ YOLOv3Loss:
   ignore_thresh: 0.7
   scale_x_y: 1.05
   label_smooth: false
-  use_fine_grained_loss: true
   iou_loss: IouLoss
   iou_aware_loss: IouAwareLoss
 
diff --git a/configs/ppyolo/ppyolo_2x.yml b/configs/ppyolo/ppyolo_2x.yml
index 8c2493372e7f5814dbaabd29d3ea78bc37647211..0ffb1ecc84ba1d997bc3fad17872197a9f91def9 100644
--- a/configs/ppyolo/ppyolo_2x.yml
+++ b/configs/ppyolo/ppyolo_2x.yml
@@ -9,14 +9,12 @@ metric: COCO
 pretrain_weights: https://paddle-imagenet-models-name.bj.bcebos.com/ResNet50_vd_ssld_pretrained.tar
 weights: output/ppyolo/model_final
 num_classes: 80
-use_fine_grained_loss: true
 use_ema: true
 ema_decay: 0.9998
 
 YOLOv3:
   backbone: ResNet
   yolo_head: YOLOv3Head
-  use_fine_grained_loss: true
 
 ResNet:
   norm_type: sync_bn
@@ -48,7 +46,6 @@ YOLOv3Loss:
   ignore_thresh: 0.7
   scale_x_y: 1.05
   label_smooth: false
-  use_fine_grained_loss: true
   iou_loss: IouLoss
   iou_aware_loss: IouAwareLoss
 
diff --git a/configs/ppyolo/ppyolo_r18vd.yml b/configs/ppyolo/ppyolo_r18vd.yml
index c054d5f5d574dc7419b07cbc8e4a22113f0778ae..fee2a55da0fe682bf65c9a04fdc7e89b013e690c 100755
--- a/configs/ppyolo/ppyolo_r18vd.yml
+++ b/configs/ppyolo/ppyolo_r18vd.yml
@@ -9,14 +9,12 @@ metric: COCO
 pretrain_weights: https://paddle-imagenet-models-name.bj.bcebos.com/ResNet18_vd_pretrained.tar
 weights: output/ppyolo_tiny/model_final
 num_classes: 80
-use_fine_grained_loss: true
 use_ema: true
 ema_decay: 0.9998
 
 YOLOv3:
   backbone: ResNet
   yolo_head: YOLOv3Head
-  use_fine_grained_loss: true
 
 ResNet:
   norm_type: sync_bn
@@ -43,7 +41,6 @@ YOLOv3Loss:
   ignore_thresh: 0.7
   scale_x_y: 1.05
   label_smooth: false
-  use_fine_grained_loss: true
   iou_loss: IouLoss
 
 IouLoss:
diff --git a/configs/ppyolo/ppyolo_test.yml b/configs/ppyolo/ppyolo_test.yml
index 840865a0b8aac1a84724b3b03b24e32435992b02..bf2de1ae6c43b4748826f12df5db64d9d2d31073 100644
--- a/configs/ppyolo/ppyolo_test.yml
+++ b/configs/ppyolo/ppyolo_test.yml
@@ -11,7 +11,6 @@ metric: COCO
 pretrain_weights: https://paddle-imagenet-models-name.bj.bcebos.com/ResNet50_vd_ssld_pretrained.tar
 weights: output/ppyolo/model_final
 num_classes: 80
-use_fine_grained_loss: true
 use_ema: true
 ema_decay: 0.9998
 save_prediction_only: True
@@ -19,7 +18,6 @@ save_prediction_only: True
 YOLOv3:
   backbone: ResNet
   yolo_head: YOLOv3Head
-  use_fine_grained_loss: true
 
 ResNet:
   norm_type: sync_bn
diff --git a/configs/yolov4/yolov4_cspdarknet.yml b/configs/yolov4/yolov4_cspdarknet.yml
index 4411b054fff7e420fd6eed6bec0d4673f3140c1b..a1e1308a3e80c8534392af4f94c7fda2ef9037dc 100644
--- a/configs/yolov4/yolov4_cspdarknet.yml
+++ b/configs/yolov4/yolov4_cspdarknet.yml
@@ -8,7 +8,6 @@ metric: COCO
 pretrain_weights: https://paddlemodels.bj.bcebos.com/object_detection/yolov4_cspdarknet.pdparams
 weights: output/yolov4_cspdarknet/model_final
 num_classes: 80
-use_fine_grained_loss: true
 save_prediction_only: True
 
 YOLOv4:
diff --git a/configs/yolov4/yolov4_cspdarknet_coco.yml b/configs/yolov4/yolov4_cspdarknet_coco.yml
index 8b4a15dc530d491c2bf985e27948d5571a5d6489..ac12e748f0c185ecf00b068171417473fad5401d 100644
--- a/configs/yolov4/yolov4_cspdarknet_coco.yml
+++ b/configs/yolov4/yolov4_cspdarknet_coco.yml
@@ -8,7 +8,6 @@ metric: COCO
 pretrain_weights: https://paddlemodels.bj.bcebos.com/object_detection/CSPDarkNet53_pretrained.pdparams
 weights: output/yolov4_cspdarknet_coco/model_final
 num_classes: 80
-use_fine_grained_loss: true
 
 YOLOv4:
   backbone: CSPDarkNet
diff --git a/configs/yolov4/yolov4_cspdarknet_voc.yml b/configs/yolov4/yolov4_cspdarknet_voc.yml
index beefaa0f153ca8f9fd0541907628a2c02854024c..9c717ec1345c46a58e9022902234246e3cc20b6e 100644
--- a/configs/yolov4/yolov4_cspdarknet_voc.yml
+++ b/configs/yolov4/yolov4_cspdarknet_voc.yml
@@ -8,7 +8,6 @@ metric: VOC
 pretrain_weights: https://paddlemodels.bj.bcebos.com/object_detection/yolov4_cspdarknet.pdparams
 weights: output/yolov4_cspdarknet_voc/model_final
 num_classes: 20
-use_fine_grained_loss: true
 
 YOLOv4:
   backbone: CSPDarkNet
diff --git a/ppdet/data/reader.py b/ppdet/data/reader.py
index 3dce62d8eff338728b2e4e009308cb4b712327a6..01ce8aecdae14911e8a04b95fe15773982032748 100644
--- a/ppdet/data/reader.py
+++ b/ppdet/data/reader.py
@@ -199,7 +199,7 @@ class Reader(object):
                  class_aware_sampling=False,
                  worker_num=-1,
                  use_process=False,
-                 use_fine_grained_loss=False,
+                 use_fine_grained_loss=True,
                  num_classes=80,
                  bufsize=-1,
                  memsize='3G',
diff --git a/ppdet/modeling/architectures/yolo.py b/ppdet/modeling/architectures/yolo.py
index 80ac34e427fb18dd70ed57a11a9d534137537096..768c5d4c0ca5d04f99f850604f916ba671d03a4d 100644
--- a/ppdet/modeling/architectures/yolo.py
+++ b/ppdet/modeling/architectures/yolo.py
@@ -43,7 +43,7 @@ class YOLOv3(object):
     def __init__(self,
                  backbone,
                  yolo_head='YOLOv3Head',
-                 use_fine_grained_loss=False):
+                 use_fine_grained_loss=True):
         super(YOLOv3, self).__init__()
         self.backbone = backbone
         self.yolo_head = yolo_head
@@ -182,7 +182,7 @@ class YOLOv4(YOLOv3):
     def __init__(self,
                  backbone,
                  yolo_head='YOLOv4Head',
-                 use_fine_grained_loss=False):
+                 use_fine_grained_loss=True):
         super(YOLOv4, self).__init__(
             backbone=backbone,
             yolo_head=yolo_head,
diff --git a/ppdet/modeling/losses/yolo_loss.py b/ppdet/modeling/losses/yolo_loss.py
index 6823c024bc5aea54c4f9b5195da03cb16dfee8a3..243737102c93a24cfbc4408891382cdc5a02725d 100644
--- a/ppdet/modeling/losses/yolo_loss.py
+++ b/ppdet/modeling/losses/yolo_loss.py
@@ -45,7 +45,7 @@ class YOLOv3Loss(object):
                  batch_size=8,
                  ignore_thresh=0.7,
                  label_smooth=True,
-                 use_fine_grained_loss=False,
+                 use_fine_grained_loss=True,
                  iou_loss=None,
                  iou_aware_loss=None,
                  downsample=[32, 16, 8],