diff --git a/configs/vitdet/cascade_rcnn_vit_large_hrfpn_cae_1x_coco.yml b/configs/vitdet/cascade_rcnn_vit_large_hrfpn_cae_1x_coco.yml
index 5f5bbfb46ea1bdb748d31b4d8ea793ebd903fa33..3bd443ef3e3fc0d2dcb623b60e45c51c629c6d2a 100644
--- a/configs/vitdet/cascade_rcnn_vit_large_hrfpn_cae_1x_coco.yml
+++ b/configs/vitdet/cascade_rcnn_vit_large_hrfpn_cae_1x_coco.yml
@@ -7,6 +7,7 @@ weights: output/cascade_rcnn_vit_large_hrfpn_cae_1x_coco/model_final
 
 depth: &depth 24
 dim: &dim 1024
+use_fused_allreduce_gradients: &use_checkpoint True
 
 VisionTransformer:
   img_size: [800, 1344]
@@ -15,6 +16,7 @@ VisionTransformer:
   num_heads: 16
   drop_path_rate: 0.25
   out_indices: [7, 11, 15, 23]
+  use_checkpoint: *use_checkpoint
   pretrained: https://bj.bcebos.com/v1/paddledet/models/pretrained/vit_large_cae_pretrained.pdparams
 
 HRFPN:
diff --git a/ppdet/modeling/backbones/vision_transformer.py b/ppdet/modeling/backbones/vision_transformer.py
index 798ea376878f09ade54e3a5c9bbc6f825769db72..26e6ff2a062c85c309a91123ee41b6d6469723ca 100644
--- a/ppdet/modeling/backbones/vision_transformer.py
+++ b/ppdet/modeling/backbones/vision_transformer.py
@@ -596,7 +596,7 @@ class VisionTransformer(nn.Layer):
 
         feats = []
         for idx, blk in enumerate(self.blocks):
-            if self.use_checkpoint:
+            if self.use_checkpoint and self.training:
                 x = paddle.distributed.fleet.utils.recompute(
                     blk, x, rel_pos_bias, **{"preserve_rng_state": True})
             else: