diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index 81d1024cb65f6209c028e922fa15cdaee18eda94..2604e41045b74af323c7ba6e5788cd06ae0d8fbc 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -197,9 +197,17 @@ ParallelExecutor::ParallelExecutor(
     PADDLE_ENFORCE(places.size() > 1,
                    "If you set build_strategy.reduce with 'Reduce',"
                    "the number of places must be greater than 1.");
-    PADDLE_ENFORCE(exec_strategy.type_ != ExecutionStrategy::kParallelGraph,
-                   "You should set build_strategy.reduce with 'AllReduce' for "
-                   "the ParallelGraph executor type");
+  }
+
+  if (exec_strategy.type_ == ExecutionStrategy::kParallelGraph) {
+    PADDLE_ENFORCE(
+        member_->use_all_reduce_,
+        "build_strategy.reduce should be `AllReduce` if you want to use"
+        "ParallelGraph executor.");
+    PADDLE_ENFORCE(
+        member_->use_cuda_,
+        "execution_strategy.use_cuda should be True if you want to use"
+        "ParallelGraph executor.");
   }
 
   // Step 1. Bcast the params to devs.
diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py
index 3dddff0d99dcb9463dbc489b88f1cdd17cc8bcf4..0ff079b4e2c7a204820f43850049f296508ff40c 100644
--- a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py
+++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py
@@ -166,6 +166,8 @@ class TestMNIST(TestParallelExecutorBase):
     def check_batchnorm_fc_convergence(self, use_cuda, exec_type):
         if use_cuda and not core.is_compiled_with_cuda():
             return
+        if not use_cuda and exec_type == ExecutorType.ParallelGraph:
+            return
 
         img, label = self._init_data()
 
diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py
index b5ee72a24e60d39ca4d85c3d10548a584b577a0a..8a1a3ab3caeed0a36d13534ffa0db7c5ba438b94 100644
--- a/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py
+++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py
@@ -173,10 +173,6 @@ class TestTransformer(TestParallelExecutorBase):
     def test_main(self):
         if core.is_compiled_with_cuda():
             self.check_network_convergence(transformer, use_cuda=True)
-            self.check_network_convergence(
-                transformer,
-                use_cuda=True,
-                exec_type=ExecutorType.ParallelGraph)
             self.check_network_convergence(
                 transformer, use_cuda=True, enable_sequential_execution=True)
         self.check_network_convergence(transformer, use_cuda=False, iter=5)