diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index 6ad86e900dd2f7aa88cfe855aff45766dbd7ef13..45e34cab4c00695ee67d176487d2030ac8928cff 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -292,10 +292,11 @@ ParallelExecutor::ParallelExecutor(
   graphs.push_back(std::move(graph));
 #endif
   auto max_memory_size = GetEagerDeletionThreshold();
-  // TODO(Yancey1989): fix gc failed on ParallelGraph strategy.
-  if (max_memory_size >= 0 && !build_strategy.enable_parallel_graph_) {
-    graphs[0] = member_->PrepareGCAndRefCnts(
-        std::move(graphs[0]), static_cast<size_t>(max_memory_size));
+  if (max_memory_size >= 0) {
+    for (size_t i = 0; i < graphs.size(); ++i) {
+      graphs[i] = member_->PrepareGCAndRefCnts(
+          std::move(graphs[i]), static_cast<size_t>(max_memory_size));
+    }
   }
 
   // Step 3. Create vars in each scope. Passes may also create new vars.