diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index d370d72c321f85d314cb7dce75e1e898d36c7ff2..ac4e346959685ad5607aefd9ef0fa885e8ae71fe 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -314,25 +314,23 @@ ParallelExecutor::ParallelExecutor(const std::vector<platform::Place> &places,
   member_->use_all_reduce_ =
       build_strategy.reduce_ == BuildStrategy::ReduceStrategy::kAllReduce;
   member_->nranks_ = build_strategy.num_trainers_ * places.size();
+  if (!member_->use_all_reduce_ && member_->nranks_ == 1) {
+    LOG(INFO) << "If you set build_strategy.reduce with 'Reduce',"
+                 "the number of places should be greater than 1.";
+    member_->build_strategy_.reduce_ =
+        BuildStrategy::ReduceStrategy::kAllReduce;
+    member_->use_all_reduce_ = true;
+  }
 #if defined(PADDLE_WITH_CUDA) && defined(_WIN32)
   if (member_->use_cuda_) {
     PADDLE_ENFORCE(places.size() == 1, "Windows can support Single GPU only.");
   }
 #endif
-  if (!member_->use_all_reduce_) {
-    if (places.size() == 1) {
-      LOG(INFO) << "If you set build_strategy.reduce with 'Reduce',"
-                   "the number of places should be greater than 1.";
-      member_->use_all_reduce_ = true;
-    }
-  }
-
   LOG(INFO) << string::Sprintf(
       "The number of %s, which is used in ParallelExecutor, is %lu. And "
       "the Program will be copied %lu copies",
       (member_->use_cuda_ ? "CUDAPlace" : "CPUPlace"), places.size(),
       places.size());
-
   // Step 1. Bcast the bcast_vars to devs.
   // Create local scopes
   if (local_scopes.empty()) {