diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index 4abde1f21e9701794154bee0c78be8c35582c34c..662a29d41937521810473cf29fe9494b1b4a9d9e 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -158,10 +158,13 @@ ParallelExecutor::ParallelExecutor(
       var_infos.back().persistable_ = node->Var()->Persistable();
     }
   }
-  // If the loss_var_name is given, the number of graph should be only one.
-  if (loss_var_name.size()) {
-    PADDLE_ENFORCE_EQ(ir::GraphNum(*graph), 1,
-                      "The number of graph should be only one");
+
+  if (VLOG_IS_ON(5)) {
+    // If the loss_var_name is given, the number of graph should be only one.
+    if (loss_var_name.size()) {
+      PADDLE_ENFORCE_EQ(ir::GraphNum(*graph), 1,
+                        "The number of graph should be only one");
+    }
   }
 
   if (exec_strategy.type_ == ExecutionStrategy::kDefault) {