diff --git a/paddle/fluid/framework/transfer_scope_cache.cc b/paddle/fluid/framework/transfer_scope_cache.cc
index e04c4583d33ca5d15eb104b08e36024a99f1085a..c812a6dc95a2551c75c696879352f430f75f2c9c 100644
--- a/paddle/fluid/framework/transfer_scope_cache.cc
+++ b/paddle/fluid/framework/transfer_scope_cache.cc
@@ -27,6 +27,13 @@ std::unordered_set<Scope*>& global_transfer_scope_cache() {
   return *x;
 }
 
+std::unordered_map<const Scope*, std::unordered_set<size_t>>&
+global_transfer_scope_key() {
+  thread_local auto* x =
+      new std::unordered_map<const Scope*, std::unordered_set<size_t>>;
+  return *x;
+}
+
 Scope* TryCreateTransferScope(OpKernelType type0,
                               OpKernelType type1,
                               const Scope* scope) {
@@ -36,6 +43,8 @@ Scope* TryCreateTransferScope(OpKernelType type0,
   infer_cache_key =
       CombineHash(infer_cache_key, std::hash<const Scope*>()(scope));
 
+  global_transfer_scope_key()[scope].insert(infer_cache_key);
+
   auto it = global_transfer_data_cache().find(infer_cache_key);
   if (it != global_transfer_data_cache().end()) {
     new_scope = global_transfer_data_cache()[infer_cache_key];
diff --git a/paddle/fluid/framework/transfer_scope_cache.h b/paddle/fluid/framework/transfer_scope_cache.h
index 7e639de615c0a293b4dc92d4c8f485f51678d0af..da2e319d5ba5e41280d3f265217a872bdf992913 100644
--- a/paddle/fluid/framework/transfer_scope_cache.h
+++ b/paddle/fluid/framework/transfer_scope_cache.h
@@ -31,6 +31,9 @@ std::unordered_map<size_t, Scope*>& global_transfer_data_cache();
 
 std::unordered_set<Scope*>& global_transfer_scope_cache();
 
+std::unordered_map<const Scope*, std::unordered_set<size_t>>&
+global_transfer_scope_key();
+
 // Combine two hash values to a single hash.
 static size_t CombineHash(size_t seed, size_t a) {
   return (seed ^ a) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc
index a8962c61e47ab89ee101588784f6f66ae3002610..49045089ce5a045ddbc4fdb1ca060ab00baa3015 100644
--- a/paddle/fluid/inference/api/analysis_predictor.cc
+++ b/paddle/fluid/inference/api/analysis_predictor.cc
@@ -33,6 +33,7 @@
 #include "paddle/fluid/framework/naive_executor.h"
 #include "paddle/fluid/framework/op_proto_maker.h"
 #include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/framework/transfer_scope_cache.h"
 #include "paddle/fluid/framework/var_type_traits.h"
 #include "paddle/fluid/framework/version.h"
 #include "paddle/fluid/inference/analysis/helper.h"
@@ -1928,6 +1929,15 @@ AnalysisPredictor::~AnalysisPredictor() {
                               "./profile.log");
   }
   if (sub_scope_) {
+    if (framework::global_transfer_scope_key().find(sub_scope_) !=
+        framework::global_transfer_scope_key().end()) {
+      auto scope_key_set = framework::global_transfer_scope_key()[sub_scope_];
+      for (auto iter = scope_key_set.begin(); iter != scope_key_set.end();
+           iter++) {
+        framework::global_transfer_data_cache().erase(*iter);
+      }
+      framework::global_transfer_scope_key().erase(sub_scope_);
+    }
     scope_->DeleteScope(sub_scope_);
   }