diff --git a/model_zoo/Transformer/train.py b/model_zoo/Transformer/train.py
index 23c0eb78fd581be2d646969f383c227614d75a7b..ffd6b8c7145d1c4d4c9b9300143b62b9681fff69 100644
--- a/model_zoo/Transformer/train.py
+++ b/model_zoo/Transformer/train.py
@@ -147,10 +147,11 @@ def run_transformer_train():
 
     callbacks = [TimeMonitor(dataset.get_dataset_size()), LossCallBack()]
     if args.enable_save_ckpt == "true":
-        ckpt_config = CheckpointConfig(save_checkpoint_steps=args.save_checkpoint_steps,
-                                       keep_checkpoint_max=args.save_checkpoint_num)
-        ckpoint_cb = ModelCheckpoint(prefix='transformer', directory=args.save_checkpoint_path, config=ckpt_config)
-        callbacks.append(ckpoint_cb)
+        if device_num == 1 or (device_num > 1 and rank_id == 0):
+            ckpt_config = CheckpointConfig(save_checkpoint_steps=args.save_checkpoint_steps,
+                                           keep_checkpoint_max=args.save_checkpoint_num)
+            ckpoint_cb = ModelCheckpoint(prefix='transformer', directory=args.save_checkpoint_path, config=ckpt_config)
+            callbacks.append(ckpoint_cb)
 
     if args.enable_lossscale == "true":
         scale_manager = DynamicLossScaleManager(init_loss_scale=cfg.init_loss_scale_value,