diff --git a/python/paddle/fluid/memory_optimization_transpiler.py b/python/paddle/fluid/memory_optimization_transpiler.py
index e6e98cbfecd78ef61a5e0b8e7972a0240904be50..41d1eca82e8b680977f44f1756c25c37340668a4 100644
--- a/python/paddle/fluid/memory_optimization_transpiler.py
+++ b/python/paddle/fluid/memory_optimization_transpiler.py
@@ -148,7 +148,6 @@ class ControlFlowGraph(object):
                 self._skip_opt.update(op.output_arg_names())
 
     def release_memory(self):
-        self._build_graph()
         self._dataflow_analyze()
         self._update_skip_opt_set()
         fwd_id = 0
@@ -187,7 +186,6 @@ class ControlFlowGraph(object):
                     return True
             return False
 
-        self._build_graph()
         self._dataflow_analyze()
         self._update_skip_opt_set()
         self.pool = []