diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc
index 64592d73e1741c2bc93a2c90b58b1824b2c887f9..c2063b5e6a0db257b2eeffde9e3b4a7a0ebfbc37 100644
--- a/paddle/fluid/framework/operator.cc
+++ b/paddle/fluid/framework/operator.cc
@@ -916,7 +916,14 @@ std::vector<KernelConfig>* OperatorWithKernel::GetKernelConfig(
 
 void OperatorWithKernel::RunImpl(const Scope& scope,
                                  const platform::Place& place) const {
-  RuntimeContext ctx(Inputs(), Outputs(), scope);
+  if (!runtime_ctx_) {
+    // RuntimeContext is used to relate input/output names of Operator with
+    // the corresponding variables in Scope.
+    // Since the input/output names of Operator do not change in the execution,
+    // RuntimeContext could be created only at the first iteration of
+    // the execution to save the elapsed time.
+    runtime_ctx_ = new RuntimeContext(Inputs(), Outputs(), scope);
+  }
   platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
   auto* dev_ctx = pool.Get(place);
 
@@ -931,7 +938,7 @@ void OperatorWithKernel::RunImpl(const Scope& scope,
   OpKernelMap& kernels = kernels_iter->second;
 
   auto expected_kernel_key = this->GetExpectedKernelType(
-      ExecutionContext(*this, scope, *dev_ctx, ctx, nullptr));
+      ExecutionContext(*this, scope, *dev_ctx, *runtime_ctx_, nullptr));
   VLOG(3) << "expected_kernel_key:" << expected_kernel_key;
 
   auto kernel_iter = kernels.find(expected_kernel_key);
@@ -955,8 +962,8 @@ void OperatorWithKernel::RunImpl(const Scope& scope,
 
   // do data transformScope &transfer_scope;
   std::vector<std::string> transfered_inplace_vars;
-  auto* transfer_scope =
-      PrepareData(scope, expected_kernel_key, &transfered_inplace_vars, &ctx);
+  auto* transfer_scope = PrepareData(scope, expected_kernel_key,
+                                     &transfered_inplace_vars, runtime_ctx_);
 
   // exec scope is the scope that kernel actually executed on.
   const Scope& exec_scope =
@@ -966,12 +973,12 @@ void OperatorWithKernel::RunImpl(const Scope& scope,
     dev_ctx = pool.Get(expected_kernel_key.place_);
   }
 
-  RuntimeInferShapeContext infer_shape_ctx(*this, exec_scope, ctx);
+  RuntimeInferShapeContext infer_shape_ctx(*this, exec_scope, *runtime_ctx_);
   this->InferShape(&infer_shape_ctx);
   // TODO(panyx0718): ExecutionContext should only depend on RuntimeContext
   // not Scope. Imperative mode only pass inputs and get outputs.
-  kernel_iter->second(
-      ExecutionContext(*this, exec_scope, *dev_ctx, ctx, kernel_configs));
+  kernel_iter->second(ExecutionContext(*this, exec_scope, *dev_ctx,
+                                       *runtime_ctx_, kernel_configs));
 
   if (!transfered_inplace_vars.empty()) {
     // there is inplace variable has been transfered.
diff --git a/paddle/fluid/framework/operator.h b/paddle/fluid/framework/operator.h
index 8a86813e9362d7b82c2023428a35a1982adb0508..e34a0e21415c3615d0ce9b85f8a17a230347949d 100644
--- a/paddle/fluid/framework/operator.h
+++ b/paddle/fluid/framework/operator.h
@@ -541,6 +541,7 @@ class OperatorWithKernel : public OperatorBase {
 
  protected:
   mutable OpKernelConfigsMap kernel_configs_map_;
+  mutable RuntimeContext* runtime_ctx_ = nullptr;
 };
 
 extern bool OpSupportGPU(const std::string& op_type);