diff --git a/paddle/fluid/framework/details/computation_op_handle.cc b/paddle/fluid/framework/details/computation_op_handle.cc
index 5867f8fc554992f3e10ba42ed1a96d913deb38fb..348b944cf921c5dd792ea11ce34eab056e2334e7 100644
--- a/paddle/fluid/framework/details/computation_op_handle.cc
+++ b/paddle/fluid/framework/details/computation_op_handle.cc
@@ -35,6 +35,8 @@ void ComputationOpHandle::RunImpl() {
 
   op_->Run(*scope_, place_);
 }
+
+std::string ComputationOpHandle::Name() const { return op_->Type(); }
 }  // namespace details
 }  // namespace framework
 }  // namespace paddle
diff --git a/paddle/fluid/framework/details/computation_op_handle.h b/paddle/fluid/framework/details/computation_op_handle.h
index 1fbfd4eabe09a41e56fe462902d9fc8b2cb6c708..d6d2d731ca80a0fbc0a2a34027b5b7c3c1977c07 100644
--- a/paddle/fluid/framework/details/computation_op_handle.h
+++ b/paddle/fluid/framework/details/computation_op_handle.h
@@ -31,6 +31,8 @@ struct ComputationOpHandle : public OpHandleBase {
   ComputationOpHandle(const OpDesc &op_desc, Scope *scope,
                       platform::Place place);
 
+  std::string Name() const override;
+
  protected:
   void RunImpl() override;
 };
diff --git a/paddle/fluid/framework/details/fetch_op_handle.cc b/paddle/fluid/framework/details/fetch_op_handle.cc
index ab552081a4ab9d03e177f75012e446286db66b05..c697a1c93786d43d540ba7c40f6fd78ffcdcb0e0 100644
--- a/paddle/fluid/framework/details/fetch_op_handle.cc
+++ b/paddle/fluid/framework/details/fetch_op_handle.cc
@@ -72,6 +72,8 @@ void FetchOpHandle::RunImpl() {
   }
 }
 
+std::string FetchOpHandle::Name() const { return "Fetch"; }
+
 }  // namespace details
 }  // namespace framework
 }  // namespace paddle
diff --git a/paddle/fluid/framework/details/fetch_op_handle.h b/paddle/fluid/framework/details/fetch_op_handle.h
index 3123f7ba2323a7885c030d18a345d6f258f894b7..904b2d669f8b156b99197afb0155380d1170a68b 100644
--- a/paddle/fluid/framework/details/fetch_op_handle.h
+++ b/paddle/fluid/framework/details/fetch_op_handle.h
@@ -38,6 +38,8 @@ struct FetchOpHandle : public OpHandleBase {
 
   void WaitAndMergeCPUTensors() const;
 
+  std::string Name() const override;
+
  protected:
   void RunImpl() override;
 };
diff --git a/paddle/fluid/framework/details/multi_devices_graph_builder.cc b/paddle/fluid/framework/details/multi_devices_graph_builder.cc
index b27647a8eebcf43b4331f946662af59596253ff0..cb02d36714d8dc7a878db0832c6a1533671f4535 100644
--- a/paddle/fluid/framework/details/multi_devices_graph_builder.cc
+++ b/paddle/fluid/framework/details/multi_devices_graph_builder.cc
@@ -136,6 +136,12 @@ std::unique_ptr<SSAGraph> MultiDevSSAGraphBuilder::Build(
    */
   PolishGraphToSupportDataHazards(&result);
 
+  if (VLOG_IS_ON(10)) {
+    std::ostringstream sout;
+    PrintGraphviz(*graph, sout);
+    VLOG(10) << sout.str();
+  }
+
   return std::unique_ptr<SSAGraph>(graph);
 }
 }  // namespace details
diff --git a/paddle/fluid/framework/details/nccl_all_reduce_op_handle.cc b/paddle/fluid/framework/details/nccl_all_reduce_op_handle.cc
index a79c61f3593f819ff8a5c7435458743017be488b..f2303ff4cabf31eb2fcf53bd24a2dfb8a7ab30a6 100644
--- a/paddle/fluid/framework/details/nccl_all_reduce_op_handle.cc
+++ b/paddle/fluid/framework/details/nccl_all_reduce_op_handle.cc
@@ -69,6 +69,8 @@ void NCCLAllReduceOpHandle::RunImpl() {
     }
   }
 }
+
+std::string NCCLAllReduceOpHandle::Name() const { return "NCCL AllReduce"; }
 }  // namespace details
 }  // namespace framework
 }  // namespace paddle
diff --git a/paddle/fluid/framework/details/nccl_all_reduce_op_handle.h b/paddle/fluid/framework/details/nccl_all_reduce_op_handle.h
index 7152d1a587e3774efb0c3a60c0d49a67b68ad74d..045070bb6a97e90600cd24d9f43cd2a10a4bc1f5 100644
--- a/paddle/fluid/framework/details/nccl_all_reduce_op_handle.h
+++ b/paddle/fluid/framework/details/nccl_all_reduce_op_handle.h
@@ -32,6 +32,8 @@ struct NCCLAllReduceOpHandle : public OpHandleBase {
                         const std::vector<platform::Place> &places,
                         const platform::NCCLContextMap &ctxs);
 
+  std::string Name() const override;
+
  protected:
   void RunImpl() override;
 };
diff --git a/paddle/fluid/framework/details/op_handle_base.h b/paddle/fluid/framework/details/op_handle_base.h
index 5178b51d8d77d44a6d25305a3a67dbf6effb7cad..99d896848675c96d806ff36258e81d346a8d9db3 100644
--- a/paddle/fluid/framework/details/op_handle_base.h
+++ b/paddle/fluid/framework/details/op_handle_base.h
@@ -33,6 +33,8 @@ struct OpHandleBase {
 
   std::string DebugString() const;
 
+  virtual std::string Name() const = 0;
+
   virtual ~OpHandleBase();
 
   void Run(bool use_event);
diff --git a/paddle/fluid/framework/details/scale_loss_grad_op_handle.cc b/paddle/fluid/framework/details/scale_loss_grad_op_handle.cc
index 2e69f1e5e84e23c6144bdfb8b9f521f4277ce7ee..a6a67c9b1452392edbda60ebcdd1b4fb4311a37b 100644
--- a/paddle/fluid/framework/details/scale_loss_grad_op_handle.cc
+++ b/paddle/fluid/framework/details/scale_loss_grad_op_handle.cc
@@ -45,6 +45,8 @@ void ScaleLossGradOpHandle::RunImpl() {
 #endif
   }
 }
+
+std::string ScaleLossGradOpHandle::Name() const { return "Scale LossGrad"; }
 }  // namespace details
 }  // namespace framework
 }  // namespace paddle
diff --git a/paddle/fluid/framework/details/scale_loss_grad_op_handle.h b/paddle/fluid/framework/details/scale_loss_grad_op_handle.h
index 3a355749192cc229e772b6cbf4cb5c2746edb630..ab7353a4fc56bebfe04696efd838dc4559218058 100644
--- a/paddle/fluid/framework/details/scale_loss_grad_op_handle.h
+++ b/paddle/fluid/framework/details/scale_loss_grad_op_handle.h
@@ -32,6 +32,8 @@ struct ScaleLossGradOpHandle : public OpHandleBase {
 
   ~ScaleLossGradOpHandle() final;
 
+  std::string Name() const override;
+
  protected:
   void RunImpl() override;
 };
diff --git a/paddle/fluid/framework/details/ssa_graph_builder.cc b/paddle/fluid/framework/details/ssa_graph_builder.cc
index 7a80a4b1e73d79d369bbb2f2fc453e1dc7e9f57c..e0209fce76b1459fd39c146da4a5e610554d3804 100644
--- a/paddle/fluid/framework/details/ssa_graph_builder.cc
+++ b/paddle/fluid/framework/details/ssa_graph_builder.cc
@@ -83,6 +83,64 @@ void SSAGraphBuilder::CreateOpOutput(SSAGraph *graph, OpHandleBase *op_handle,
   var.place_ = place;
   op_handle->AddOutput(&var);
 }
+
+template <typename Callback>
+void IterAllVar(const SSAGraph &graph, Callback callback) {
+  for (auto &each : graph.vars_) {
+    for (auto &pair1 : each) {
+      for (auto &pair2 : pair1.second) {
+        callback(pair2.second);
+      }
+    }
+  }
+
+  for (auto &var : graph.dep_vars_) {
+    callback(*var);
+  }
+}
+
+void SSAGraphBuilder::PrintGraphviz(const SSAGraph &graph, std::ostream &sout) {
+  size_t var_id = 0;
+  std::unordered_map<const VarHandleBase *, size_t> vars;
+
+  sout << "digraph G {\n";
+
+  IterAllVar(graph, [&](const VarHandleBase &var) {
+    auto *var_ptr = &var;
+    auto *var_handle_ptr = dynamic_cast<const VarHandle *>(var_ptr);
+    auto *dummy_ptr = dynamic_cast<const DummyVarHandle *>(var_ptr);
+
+    size_t cur_var_id = var_id++;
+    vars[var_ptr] = cur_var_id;
+
+    if (var_handle_ptr) {
+      sout << "var_" << cur_var_id << " [label=\"" << var_handle_ptr->name_
+           << "\\n"
+           << var_handle_ptr->place_ << "\\n"
+           << var_handle_ptr->version_ << "\"]" << std::endl;
+    } else if (dummy_ptr) {
+      sout << "var_" << cur_var_id << " [label=\"dummy\"]" << std::endl;
+    }
+  });
+
+  size_t op_id = 0;
+  for (auto &op : graph.ops_) {
+    std::string op_name = "op_" + std::to_string(op_id++);
+    sout << op_name << " [label=\"" << op->Name() << "\", shape=rect]"
+         << std::endl;
+    for (auto in : op->inputs_) {
+      std::string var_name = "var_" + std::to_string(vars[in]);
+      sout << var_name << " -> " << op_name << std::endl;
+    }
+
+    for (auto out : op->outputs_) {
+      std::string var_name = "var_" + std::to_string(vars[out]);
+      sout << op_name << " -> " << var_name << std::endl;
+    }
+  }
+
+  sout << "}\n";
+}
 }  // namespace details
 }  // namespace framework
 }  // namespace paddle
diff --git a/paddle/fluid/framework/details/ssa_graph_builder.h b/paddle/fluid/framework/details/ssa_graph_builder.h
index df05bb739421672fd8b169b45246b9a75f77ca7f..bf20e7164a100718c1dcfe3ef971cfff60bbbaa2 100644
--- a/paddle/fluid/framework/details/ssa_graph_builder.h
+++ b/paddle/fluid/framework/details/ssa_graph_builder.h
@@ -51,6 +51,8 @@ class SSAGraphBuilder {
   static void CreateOpOutput(SSAGraph *graph, OpHandleBase *op_handle,
                              const std::string &each_var_name,
                              const platform::Place &place, size_t place_offset);
+
+  static void PrintGraphviz(const SSAGraph &graph, std::ostream &sout);
 };
 }  // namespace details
 }  // namespace framework
diff --git a/paddle/fluid/framework/details/threaded_ssa_graph_executor.cc b/paddle/fluid/framework/details/threaded_ssa_graph_executor.cc
index 86e880ed72e5c2217d77e74769c4dab468f17e69..f609395d40f70756ff6ac5a797cee103f424be2e 100644
--- a/paddle/fluid/framework/details/threaded_ssa_graph_executor.cc
+++ b/paddle/fluid/framework/details/threaded_ssa_graph_executor.cc
@@ -133,6 +133,12 @@ FeedFetchList ThreadedSSAGraphExecutor::Run(
       if (exception_) {
         throw * exception_;
       }
+
+      VLOG(10) << "=============================";
+      for (auto &op : pending_ops) {
+        VLOG(10) << op.first->DebugString();
+      }
+
       // keep waiting the ready variables
       continue;
     }
diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor.py b/python/paddle/fluid/tests/unittests/test_parallel_executor.py
index 2ebdbaaca65fe865f6c24f9614674b6d18eba0e7..dd6e70eadbd83603ac093810d74fd1853195891f 100644
--- a/python/paddle/fluid/tests/unittests/test_parallel_executor.py
+++ b/python/paddle/fluid/tests/unittests/test_parallel_executor.py
@@ -48,7 +48,7 @@ def fc_with_batchnorm():
         dtypes=['float32', 'int64'])
     img, label = fluid.layers.read_file(reader)
     hidden = img
-    for _ in xrange(4):
+    for _ in xrange(1):
         hidden = fluid.layers.fc(
             hidden,
             size=200,