diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc
index 47b75228cdbd2a8b4f0c5ad33aa82f5e43044606..ba970254e590308851e31542e2037e9c5590c7fa 100644
--- a/paddle/framework/block_desc.cc
+++ b/paddle/framework/block_desc.cc
@@ -72,13 +72,13 @@ std::vector<OpDescBind *> BlockDescBind::AllOps() const {
 void BlockDescBind::Flush() {
   if (need_update_) {
     auto &op_field = *this->desc_->mutable_ops();
-    op_field.Clear();
+    this->ClearPBOps();
     op_field.Reserve(static_cast<int>(ops_.size()));
     for (auto &op_desc : ops_) {
       op_field.AddAllocated(op_desc->Proto());
     }
     auto &var_field = *this->desc_->mutable_vars();
-    var_field.Clear();
+    this->ClearPBVars();
     var_field.Reserve(static_cast<int>(vars_.size()));
     for (auto &var_desc : vars_) {
       var_field.AddAllocated(var_desc.second->Proto());
@@ -99,5 +99,21 @@ BlockDesc *BlockDescBind::Proto() {
   return desc_;
 }
 
+void BlockDescBind::ClearPBOps() {
+  auto ops = this->desc_->mutable_ops();
+  while (!ops->empty()) {
+    // we do not own the OpDesc, so release the ownership.
+    ops->ReleaseLast();
+  }
+}
+
+void BlockDescBind::ClearPBVars() {
+  auto vars = this->desc_->mutable_vars();
+  while (!vars->empty()) {
+    // we do not own the VarDesc, so release the ownership.
+    vars->ReleaseLast();
+  }
+}
+
 }  // namespace framework
 }  // namespace paddle
diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h
index 9fb88f963283c72e1ec389b72dd2d98049c74f6d..dd7b1228befd6b4f19c9aa3dd493b0d8005359fc 100644
--- a/paddle/framework/block_desc.h
+++ b/paddle/framework/block_desc.h
@@ -36,6 +36,11 @@ class BlockDescBind {
   BlockDescBind(ProgramDescBind *prog, BlockDesc *desc)
       : prog_(prog), desc_(desc), need_update_(false) {}
 
+  ~BlockDescBind() {
+    this->ClearPBVars();
+    this->ClearPBOps();
+  }
+
   int32_t ID() const { return desc_->idx(); }
 
   int32_t Parent() const { return desc_->parent_idx(); }
@@ -60,6 +65,10 @@ class BlockDescBind {
 
   BlockDesc *Proto();
 
+ private:
+  void ClearPBOps();
+  void ClearPBVars();
+
   // FIXME(yuyang18): backward will access private data of BlockDesc.
   // Mark it public temporary. We can fix it later.
  public: