diff --git a/python/paddle/fluid/transpiler/distribute_transpiler.py b/python/paddle/fluid/transpiler/distribute_transpiler.py
index dc9d254fa55311c828affd8bdcffc16af1fc29cc..1d51ed45792664803e7ede3aa9e20a30f938e66d 100644
--- a/python/paddle/fluid/transpiler/distribute_transpiler.py
+++ b/python/paddle/fluid/transpiler/distribute_transpiler.py
@@ -543,6 +543,11 @@ class DistributeTranspiler:
 
         serial_number = self._get_lastest_checkpoint_dir(self.checkpoint_dir)
 
+        startup_prog.global_block().create_var(
+            name=SERIAL_VAR_NAME,
+            persistable=True,
+            type=core.VarDesc.VarType.RAW)
+
         startup_prog.global_block().append_op(
             type="checkpoint_load",
             inputs={"X": load_vars},
@@ -612,6 +617,11 @@ class DistributeTranspiler:
 
         serial_number = self._get_lastest_checkpoint_dir(self.checkpoint_dir)
 
+        s_prog.global_block().create_var(
+            name=SERIAL_VAR_NAME,
+            persistable=True,
+            type=core.VarDesc.VarType.RAW)
+
         s_prog.global_block().append_op(
             type="checkpoint_load",
             inputs={"X": load_vars},