diff --git a/paddle/fluid/framework/details/build_strategy.h b/paddle/fluid/framework/details/build_strategy.h
index b67a313df6c91d294314062b4049dc1d00f1ae8e..929cb51b8454b0220c5dbe6a7b82af6af06c1d53 100644
--- a/paddle/fluid/framework/details/build_strategy.h
+++ b/paddle/fluid/framework/details/build_strategy.h
@@ -89,7 +89,7 @@ struct BuildStrategy {
   bool fuse_elewise_add_act_ops_{false};
   // Fuse_all_optimizer_ops and fuse_all_reduce_ops require that gradients
   // should not be sparse types
-  bool fuse_all_optimizer_ops_{false};
+  bool fuse_all_optimizer_ops_{true};
   bool fuse_all_reduce_ops_{false};
   // fuse_relu_depthwise_conv can fuse the `relu ->
   // depthwise_conv`
diff --git a/paddle/fluid/framework/ir/fuse_optimizer_ops_pass/fuse_adam_op_pass.cc b/paddle/fluid/framework/ir/fuse_optimizer_ops_pass/fuse_adam_op_pass.cc
index 1dc2474fe1e431f552d8159b71043dbbef7e6007..997f18d70999f2a7f3c5e7214c8ddcd027101b4f 100644
--- a/paddle/fluid/framework/ir/fuse_optimizer_ops_pass/fuse_adam_op_pass.cc
+++ b/paddle/fluid/framework/ir/fuse_optimizer_ops_pass/fuse_adam_op_pass.cc
@@ -65,7 +65,8 @@ class FuseAdamOpPass : public FuseOptimizerOpPass {
 
     for (auto &node : not_need_ctrl_var_nodes) {
       // remove this node from the input op node.
-      PADDLE_ENFORCE(!node->inputs.empty());
+      PADDLE_ENFORCE(!node->inputs.empty(),
+                     "The input should not be empty here.");
       auto op_node = node->inputs.front();
       PADDLE_ENFORCE(op_node->IsOp());
       op_node->outputs.erase(