diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc
index 22c8c83f1378bf489582660713b0e5fd144f54e8..40390d41502ddf073a6b6220d3a644e8257e1fea 100644
--- a/paddle/framework/backward.cc
+++ b/paddle/framework/backward.cc
@@ -36,9 +36,11 @@ static inline std::unique_ptr<OperatorBase> CreateGradOp(
   auto grad_descs = info.grad_op_maker_(op_desc);
   std::vector<std::unique_ptr<OperatorBase>> grad_ops;
   grad_ops.reserve(grad_descs.size());
-  std::transform(
-      grad_descs.begin(), grad_descs.end(), std::back_inserter(grad_ops),
-      [](OpDescBind& grad_desc) { return OpRegistry::CreateOp(&grad_desc); });
+  std::transform(grad_descs.begin(), grad_descs.end(),
+                 std::back_inserter(grad_ops),
+                 [](const std::unique_ptr<OpDescBind>& grad_desc) {
+                   return OpRegistry::CreateOp(grad_desc.get());
+                 });
   PADDLE_ENFORCE_GT(grad_ops.size(), 0);
   if (grad_ops.size() == 1) {
     return std::move(grad_ops[0]);
diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc
index c88e85f8c4c551233a1eab0ab3546dd36758e364..830d0427fa1368f4c11a9e3164caafa7e3891ff8 100644
--- a/paddle/framework/backward_test.cc
+++ b/paddle/framework/backward_test.cc
@@ -39,13 +39,13 @@ class RowWiseAddGradMaker : public SingleGradOpDescMaker {
   using SingleGradOpDescMaker::SingleGradOpDescMaker;
 
  protected:
-  OpDescBind Apply() const override {
-    OpDescBind grad_op;
-    grad_op.SetInput(GradVarName("Out"), OutputGrad("Out"));
-    grad_op.SetOutput(GradVarName("X"), InputGrad("X"));
-    grad_op.SetOutput(GradVarName("b"), InputGrad("b"));
-    grad_op.SetType("rowwise_add_grad");
-    return grad_op;
+  std::unique_ptr<OpDescBind> Apply() const override {
+    auto grad_op = new OpDescBind();
+    grad_op->SetInput(GradVarName("Out"), OutputGrad("Out"));
+    grad_op->SetOutput(GradVarName("X"), InputGrad("X"));
+    grad_op->SetOutput(GradVarName("b"), InputGrad("b"));
+    grad_op->SetType("rowwise_add_grad");
+    return std::unique_ptr<OpDescBind>(grad_op);
   }
 };
 
@@ -147,10 +147,8 @@ class SumOpMaker : public framework::OpProtoAndCheckerMaker {
  public:
   SumOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
-    AddInput("X", "the input tensors of sum operator.")
-        .AsDuplicable()
-        .NotInGradient();
-    AddOutput("Out", "the output tensor of sum operator.").NotInGradient();
+    AddInput("X", "the input tensors of sum operator.").AsDuplicable();
+    AddOutput("Out", "the output tensor of sum operator.");
     AddComment("");
   }
 };
diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc
index 0c84cbb5a79eb9c7bb79c93ca88fc331d8e029d8..339c089e87b71efc9af466d4ea5fadb1ac68c8c2 100644
--- a/paddle/operators/mean_op.cc
+++ b/paddle/operators/mean_op.cc
@@ -57,13 +57,13 @@ class MeanGradMaker : public framework::SingleGradOpDescMaker {
   using framework::SingleGradOpDescMaker::SingleGradOpDescMaker;
 
  protected:
-  framework::OpDescBind Apply() const override {
-    framework::OpDescBind grad_op;
-    grad_op.SetType("mean_grad");
-    grad_op.SetInput("X", Input("X"));
-    grad_op.SetInput(framework::GradVarName("Out"), OutputGrad("Out"));
-    grad_op.SetOutput(framework::GradVarName("X"), InputGrad("X"));
-    return grad_op;
+  std::unique_ptr<framework::OpDescBind> Apply() const override {
+    auto* grad_op = new framework::OpDescBind();
+    grad_op->SetType("mean_grad");
+    grad_op->SetInput("X", Input("X"));
+    grad_op->SetInput(framework::GradVarName("Out"), OutputGrad("Out"));
+    grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X"));
+    return std::unique_ptr<framework::OpDescBind>(grad_op);
   }
 };
 
diff --git a/paddle/operators/minus_op.cc b/paddle/operators/minus_op.cc
index 1b3ae9a9a602a2685ee152eac21cd2167fc815af..aced8636b96d139c69895d7eba11892f190e2173 100644
--- a/paddle/operators/minus_op.cc
+++ b/paddle/operators/minus_op.cc
@@ -69,19 +69,22 @@ class MinusGradMaker : public framework::GradOpDescMakerBase {
  public:
   using framework::GradOpDescMakerBase::GradOpDescMakerBase;
 
-  std::vector<framework::OpDescBind> operator()() const override {
-    std::vector<framework::OpDescBind> ops;
+  std::vector<std::unique_ptr<framework::OpDescBind>> operator()()
+      const override {
+    std::vector<std::unique_ptr<framework::OpDescBind>> ops;
     ops.resize(2);
 
-    ops[0].SetType("scale");
-    ops[0].SetInput("X", OutputGrad("Out"));
-    ops[0].SetOutput("Out", InputGrad("X"));
-    ops[0].SetAttr("scale", 1.0f);
-
-    ops[1].SetType("scale");
-    ops[1].SetInput("X", OutputGrad("Out"));
-    ops[1].SetOutput("Out", InputGrad("Y"));
-    ops[1].SetAttr("scale", -1.0f);
+    ops[0].reset(new framework::OpDescBind());
+    ops[0]->SetType("scale");
+    ops[0]->SetInput("X", OutputGrad("Out"));
+    ops[0]->SetOutput("Out", InputGrad("X"));
+    ops[0]->SetAttr("scale", 1.0f);
+
+    ops[1].reset(new framework::OpDescBind());
+    ops[1]->SetType("scale");
+    ops[1]->SetInput("X", OutputGrad("Out"));
+    ops[1]->SetOutput("Out", InputGrad("Y"));
+    ops[1]->SetAttr("scale", -1.0f);
     return ops;
   }
 };
diff --git a/paddle/operators/pad_op.cc b/paddle/operators/pad_op.cc
index 4bd25fa46a9eb9fcd4e4be55cc12df70ca1d5c7f..944591773929bf97ad4bfc8961f58c17e424b71d 100644
--- a/paddle/operators/pad_op.cc
+++ b/paddle/operators/pad_op.cc
@@ -111,18 +111,18 @@ class PadOpGrad : public framework::OperatorWithKernel {
 };
 
 class PadOpGradMaker : public framework::SingleGradOpDescMaker {
- protected:
-  framework::OpDescBind Apply() const override {
-    framework::OpDescBind bind;
-    bind.SetInput("X", Input("X"));
-    bind.SetInput(framework::GradVarName("Out"), OutputGrad("Out"));
-    bind.SetOutput(framework::GradVarName("X"), InputGrad("X"));
-    bind.SetAttrMap(Attrs());
-    return bind;
-  }
-
  public:
   using framework::SingleGradOpDescMaker::SingleGradOpDescMaker;
+
+ protected:
+  std::unique_ptr<framework::OpDescBind> Apply() const override {
+    auto* bind = new framework::OpDescBind();
+    bind->SetInput("X", Input("X"));
+    bind->SetInput(framework::GradVarName("Out"), OutputGrad("Out"));
+    bind->SetOutput(framework::GradVarName("X"), InputGrad("X"));
+    bind->SetAttrMap(Attrs());
+    return std::unique_ptr<framework::OpDescBind>(bind);
+  }
 };
 
 }  // namespace operators
diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc
index 40f0960923260b9d593d0ddc0fb1edc46238e022..e225aecc270bc17c535c10253c970b888c42e5d3 100644
--- a/paddle/operators/scale_op.cc
+++ b/paddle/operators/scale_op.cc
@@ -57,13 +57,13 @@ class ScaleGradMaker : public framework::SingleGradOpDescMaker {
   using framework::SingleGradOpDescMaker::SingleGradOpDescMaker;
 
  protected:
-  framework::OpDescBind Apply() const override {
-    framework::OpDescBind grad_op;
-    grad_op.SetType("scale");
-    grad_op.SetInput("X", OutputGrad("Out"));
-    grad_op.SetOutput("Out", InputGrad("X"));
-    grad_op.SetAttr("scale", GetAttr("scale"));
-    return grad_op;
+  std::unique_ptr<framework::OpDescBind> Apply() const override {
+    auto *grad_op = new framework::OpDescBind();
+    grad_op->SetType("scale");
+    grad_op->SetInput("X", OutputGrad("Out"));
+    grad_op->SetOutput("Out", InputGrad("X"));
+    grad_op->SetAttr("scale", GetAttr("scale"));
+    return std::unique_ptr<framework::OpDescBind>(grad_op);
   }
 };
 
diff --git a/paddle/operators/softmax_with_cross_entropy_op.cc b/paddle/operators/softmax_with_cross_entropy_op.cc
index 87dcc3f24067e6d643e929f26db16096ed695df1..bc9868874da259a640b3bb0f38cb4ef08fc0fb83 100644
--- a/paddle/operators/softmax_with_cross_entropy_op.cc
+++ b/paddle/operators/softmax_with_cross_entropy_op.cc
@@ -167,17 +167,17 @@ class SoftmaxGradMaker : public framework::SingleGradOpDescMaker {
   using framework::SingleGradOpDescMaker::SingleGradOpDescMaker;
 
  protected:
-  framework::OpDescBind Apply() const override {
-    framework::OpDescBind grad_op;
-    grad_op.SetType("softmax_with_cross_entropy_grad");
-    grad_op.SetInput("Label", Input("Label"));
-    grad_op.SetInput("Softmax", Output("Softmax"));
-    grad_op.SetInput("Loss", Output("Loss"));
-    grad_op.SetInput(framework::GradVarName("Softmax"), OutputGrad("Softmax"));
-    grad_op.SetInput(framework::GradVarName("Loss"), OutputGrad("Loss"));
-    grad_op.SetOutput(framework::GradVarName("Logits"), InputGrad("Logits"));
-    grad_op.SetAttrMap(Attrs());
-    return grad_op;
+  std::unique_ptr<framework::OpDescBind> Apply() const override {
+    auto* grad_op = new framework::OpDescBind();
+    grad_op->SetType("softmax_with_cross_entropy_grad");
+    grad_op->SetInput("Label", Input("Label"));
+    grad_op->SetInput("Softmax", Output("Softmax"));
+    grad_op->SetInput("Loss", Output("Loss"));
+    grad_op->SetInput(framework::GradVarName("Softmax"), OutputGrad("Softmax"));
+    grad_op->SetInput(framework::GradVarName("Loss"), OutputGrad("Loss"));
+    grad_op->SetOutput(framework::GradVarName("Logits"), InputGrad("Logits"));
+    grad_op->SetAttrMap(Attrs());
+    return std::unique_ptr<framework::OpDescBind>(grad_op);
   }
 };
 
diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc
index 5ae13492b3df6a421d301777b11723ea9ac983ec..c701ee8dde26f0fd50fae227d3f345df2d7a119d 100644
--- a/paddle/operators/sum_op.cc
+++ b/paddle/operators/sum_op.cc
@@ -60,19 +60,20 @@ class SumGradMaker : public framework::GradOpDescMakerBase {
  public:
   using framework::GradOpDescMakerBase::GradOpDescMakerBase;
 
-  std::vector<framework::OpDescBind> operator()() const override {
+  std::vector<std::unique_ptr<framework::OpDescBind>> operator()()
+      const override {
     auto x_grads = InputGrad("X");
-    std::vector<framework::OpDescBind> grad_ops;
+    std::vector<std::unique_ptr<framework::OpDescBind>> grad_ops;
     grad_ops.reserve(x_grads.size());
     auto og = OutputGrad("Out");
     std::transform(x_grads.begin(), x_grads.end(), std::back_inserter(grad_ops),
                    [&og](const std::string& x_grad) {
-                     framework::OpDescBind grad_op;
-                     grad_op.SetType("scale");
-                     grad_op.SetInput("X", og);
-                     grad_op.SetOutput("Out", {x_grad});
-                     grad_op.SetAttr("scale", 1.0f);
-                     return grad_op;
+                     auto* grad_op = new framework::OpDescBind();
+                     grad_op->SetType("scale");
+                     grad_op->SetInput("X", og);
+                     grad_op->SetOutput("Out", {x_grad});
+                     grad_op->SetAttr("scale", 1.0f);
+                     return std::unique_ptr<framework::OpDescBind>(grad_op);
                    });
     return grad_ops;
   }