diff --git a/paddle/fluid/framework/details/build_strategy.cc b/paddle/fluid/framework/details/build_strategy.cc
index 51ce9732722efa44d2489f5b77694094e58c8775..a81f2842680c7e17ff72d9d4b901ae4759e37bd1 100644
--- a/paddle/fluid/framework/details/build_strategy.cc
+++ b/paddle/fluid/framework/details/build_strategy.cc
@@ -133,12 +133,15 @@ class ParallelExecutorPassBuilder : public ir::PassBuilder {
   void AppendMultiDevPass(const BuildStrategy &strategy) {
     ir::Pass *multi_devices_pass;
     if (strategy_.is_distribution_) {
+      VLOG(3) << "dist train mode";
       multi_devices_pass = AppendPass("dist_multi_devices_pass").get();
     } else {
       if (strategy.reduce_ == BuildStrategy::ReduceStrategy::kAllReduce) {
+        VLOG(3) << "allreduce mode";
         multi_devices_pass =
             AppendPass("allreduce_mode_multi_devices_pass").get();
       } else if (strategy.reduce_ == BuildStrategy::ReduceStrategy::kReduce) {
+        VLOG(3) << "reduce mode";
         multi_devices_pass = AppendPass("reduce_mode_multi_devices_pass").get();
       } else {
         PADDLE_THROW("Unknown reduce strategy.");
diff --git a/python/paddle/fluid/compiler.py b/python/paddle/fluid/compiler.py
index ef0242942838fcca737a10fafbafa61bf520b532..1c194830e1b68a655865f4553ea139f199d1122f 100644
--- a/python/paddle/fluid/compiler.py
+++ b/python/paddle/fluid/compiler.py
@@ -19,6 +19,7 @@ import sys
 from .. import compat as cpt
 
 from . import core
+from . import framework
 
 __all__ = ['CompiledProgram', 'ExecutionStrategy', 'BuildStrategy']
 
@@ -34,6 +35,15 @@ def _place_obj(place):
     return p
 
 
+def _is_pserver_mode(main_program):
+    main = main_program if main_program \
+        else framework.default_main_program()
+    for op in main.global_block().ops:
+        if op.type in ["send", "recv"]:
+            return True
+    return False
+
+
 class CompiledProgram(object):
     """
     Compiles a Program for execution.
@@ -110,6 +120,8 @@ class CompiledProgram(object):
             self._exec_strategy = ExecutionStrategy()
         if self._build_strategy is None:
             self._build_strategy = BuildStrategy()
+        self._build_strategy.is_distribution = _is_pserver_mode(
+            self._program) or self._build_strategy.num_trainers > 1
         return self
 
     def with_inference_optimize(self, config):
@@ -185,8 +197,7 @@ class CompiledProgram(object):
             self._build_strategy.trainers_endpoints = trainers_endpoints
 
         self._persistable_vars = set([
-            cpt.to_text(v.name)
-            for v in [
+            cpt.to_text(v.name) for v in [
                 var for var in self._program.list_vars()
                 if var.persistable and var.type != core.VarDesc.VarType.RAW
             ]