diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc
index 78410c0d094993a4b809fc608fd9b1fe9a28bca2..de8766809c66a92edaab41c52d8b233229ccc3ba 100644
--- a/paddle/fluid/framework/operator.cc
+++ b/paddle/fluid/framework/operator.cc
@@ -1023,6 +1023,7 @@ Scope* OperatorWithKernel::PrepareData(
     std::vector<std::string>* transfered_inplace_vars,
     RuntimeContext* ctx) const {
   Scope* new_scope = nullptr;
+  if (!need_prepare_data_) return new_scope;
 
   std::unordered_set<std::string> no_buffer_ins;
   if (info_) {
@@ -1115,6 +1116,10 @@ Scope* OperatorWithKernel::PrepareData(
       SetTensorToVariable(*var, out, trans_var);
     }
   }
+  // If new_scope = nullptr, it means that for each input of this Op, there is
+  // no TransformData. Thus, PrepareData could be skipped at the rest iterations
+  // of this Op's execution to save the elapsed time.
+  if (!new_scope) need_prepare_data_ = false;
 
   return new_scope;
 }
diff --git a/paddle/fluid/framework/operator.h b/paddle/fluid/framework/operator.h
index 489b66099658d522fe1f1adaad763b66bdd22c91..d94326563fa9ec4b532927d8474d67f9a4941d44 100644
--- a/paddle/fluid/framework/operator.h
+++ b/paddle/fluid/framework/operator.h
@@ -506,6 +506,7 @@ class OperatorWithKernel : public OperatorBase {
   mutable std::unique_ptr<OpKernelFunc> kernel_func_;
   mutable std::unique_ptr<RuntimeContext> runtime_ctx_;
   mutable const Scope* pre_scope_ = nullptr;
+  mutable bool need_prepare_data_ = true;
   mutable bool enable_cache_runtime_context = false;
   mutable bool enable_cache_expected_kernel = false;
   mutable bool all_kernels_must_compute_runtime_shape = false;