diff --git a/configs/vitdet/cascade_rcnn_vit_base_hrfpn_cae_1x_coco.yml b/configs/vitdet/cascade_rcnn_vit_base_hrfpn_cae_1x_coco.yml
index 081e371e2099aa189a1a4cc784a5f999a69aa59c..23f766d75aad37f238cfebc233941d36b2d9a295 100644
--- a/configs/vitdet/cascade_rcnn_vit_base_hrfpn_cae_1x_coco.yml
+++ b/configs/vitdet/cascade_rcnn_vit_base_hrfpn_cae_1x_coco.yml
@@ -98,8 +98,6 @@ CascadeHead:
   reg_class_agnostic: False
   stage_loss_weights: [1, 0.5, 0.25]
   loss_normalize_pos: True
-  add_gt_as_proposals: [True, True, True]
-
 
 BBoxAssigner:
   batch_size_per_im: 512
diff --git a/ppdet/modeling/heads/cascade_head.py b/ppdet/modeling/heads/cascade_head.py
index bb0beadbd38f2c2c34a730cfb1705058f3f538bd..6fdff26d3d06472ee85454222fb9be6a22873554 100644
--- a/ppdet/modeling/heads/cascade_head.py
+++ b/ppdet/modeling/heads/cascade_head.py
@@ -163,8 +163,7 @@ class CascadeHead(BBoxHead):
                  bbox_loss=None,
                  reg_class_agnostic=True,
                  stage_loss_weights=None,
-                 loss_normalize_pos=False,
-                 add_gt_as_proposals=[True, False, False]):
+                 loss_normalize_pos=False):
 
         nn.Layer.__init__(self, )
         self.head = head
@@ -180,8 +179,6 @@ class CascadeHead(BBoxHead):
         self.stage_loss_weights = [
             1. / num_cascade_stages for _ in range(num_cascade_stages)
         ] if stage_loss_weights is None else stage_loss_weights
-        self.add_gt_as_proposals = add_gt_as_proposals
-
         assert len(
             self.stage_loss_weights
         ) == num_cascade_stages, f'stage_loss_weights({len(self.stage_loss_weights)}) do not equal to num_cascade_stages({num_cascade_stages})'
@@ -224,11 +221,7 @@ class CascadeHead(BBoxHead):
         """
         targets = []
         if self.training:
-            rois, rois_num, targets = self.bbox_assigner(
-                rois,
-                rois_num,
-                inputs,
-                add_gt_as_proposals=self.add_gt_as_proposals[0])
+            rois, rois_num, targets = self.bbox_assigner(rois, rois_num, inputs)
             targets_list = [targets]
             self.assigned_rois = (rois, rois_num)
             self.assigned_targets = targets
@@ -241,12 +234,7 @@ class CascadeHead(BBoxHead):
                                                            inputs['im_shape'])
                 if self.training:
                     rois, rois_num, targets = self.bbox_assigner(
-                        rois,
-                        rois_num,
-                        inputs,
-                        i,
-                        is_cascade=True,
-                        add_gt_as_proposals=self.add_gt_as_proposals[i])
+                        rois, rois_num, inputs, i, is_cascade=True)
                     targets_list.append(targets)
 
             rois_feat = self.roi_extractor(body_feats, rois, rois_num)
@@ -318,7 +306,7 @@ class CascadeHead(BBoxHead):
         # num_or_sections in paddle.split does not support LoDTensorArray,
         # so we use [-1] to replace it if num_prop is not list. The modification
         # This ensures the correctness of both dynamic and static graphs.
-        if not isinstance(num_prop, list):
+        if not isinstance(num_prop, list): 
             num_prop = [-1]
         return pred_bbox.split(num_prop)
 
diff --git a/ppdet/modeling/proposal_generator/target.py b/ppdet/modeling/proposal_generator/target.py
index 9fbc3d48634766410c7057354484617528c5d596..fd04f052219a00c919b945d8838436de018af873 100644
--- a/ppdet/modeling/proposal_generator/target.py
+++ b/ppdet/modeling/proposal_generator/target.py
@@ -186,8 +186,7 @@ def generate_proposal_target(rpn_rois,
                              use_random=True,
                              is_cascade=False,
                              cascade_iou=0.5,
-                             assign_on_cpu=False,
-                             add_gt_as_proposals=True):
+                             assign_on_cpu=False):
 
     rois_with_gt = []
     tgt_labels = []
@@ -205,7 +204,7 @@ def generate_proposal_target(rpn_rois,
         gt_class = paddle.squeeze(gt_classes[i], axis=-1)
 
         # Concat RoIs and gt boxes except cascade rcnn or none gt
-        if add_gt_as_proposals and gt_bbox.shape[0] > 0:
+        if not is_cascade and gt_bbox.shape[0] > 0:
             bbox = paddle.concat([rpn_roi, gt_bbox])
         else:
             bbox = rpn_roi
diff --git a/ppdet/modeling/proposal_generator/target_layer.py b/ppdet/modeling/proposal_generator/target_layer.py
index c010c819de1b059a396019685f431ac822be8868..edcf973590763a27dafb4b0c7d2fa52ff7d2fa0a 100644
--- a/ppdet/modeling/proposal_generator/target_layer.py
+++ b/ppdet/modeling/proposal_generator/target_layer.py
@@ -156,8 +156,7 @@ class BBoxAssigner(object):
                  rpn_rois_num,
                  inputs,
                  stage=0,
-                 is_cascade=False,
-                 add_gt_as_proposals=True):
+                 is_cascade=False):
         gt_classes = inputs['gt_class']
         gt_boxes = inputs['gt_bbox']
         is_crowd = inputs.get('is_crowd', None)
@@ -167,7 +166,7 @@ class BBoxAssigner(object):
             rpn_rois, gt_classes, gt_boxes, self.batch_size_per_im,
             self.fg_fraction, self.fg_thresh, self.bg_thresh, self.num_classes,
             self.ignore_thresh, is_crowd, self.use_random, is_cascade,
-            self.cascade_iou[stage], self.assign_on_cpu, add_gt_as_proposals)
+            self.cascade_iou[stage], self.assign_on_cpu)
         rois = outs[0]
         rois_num = outs[-1]
         # tgt_labels, tgt_bboxes, tgt_gt_inds