diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc
index 48a140aa1135b6bf45f229c30b4028e99ed9dd96..885d797d66b48ca138fd9e992c2f71a792f11a0e 100644
--- a/paddle/fluid/inference/api/paddle_pass_builder.cc
+++ b/paddle/fluid/inference/api/paddle_pass_builder.cc
@@ -86,8 +86,7 @@ const std::vector<std::string> kAnakinSubgraphPasses({
 
 GpuPassStrategy::GpuPassStrategy() : PassStrategy({}) {
   passes_.assign({
-    "infer_clean_graph_pass",          //
-        "runtime_context_cache_pass",  //
+    "infer_clean_graph_pass",  //
         //   "identity_scale_op_clean_pass",              //
         "conv_affine_channel_fuse_pass",             //
         "conv_eltwiseadd_affine_channel_fuse_pass",  //
@@ -116,11 +115,7 @@ CpuPassStrategy::CpuPassStrategy() : PassStrategy({}) {
   // NOTE the large fusions should be located in the front, so that they will
   // not be damaged by smaller ones.
   passes_.assign({
-      "infer_clean_graph_pass",  //
-      // TODO(luotao): runtime_context_cache_pass should be located in the
-      // front, see https://github.com/PaddlePaddle/Paddle/issues/16609,
-      // will enhance this pass later.
-      "runtime_context_cache_pass",     //
+      "infer_clean_graph_pass",         //
       "attention_lstm_fuse_pass",       //
       "seqpool_concat_fuse_pass",       //
       "seqconv_eltadd_relu_fuse_pass",  //
diff --git a/paddle/fluid/inference/tests/api/analyzer_pyramid_dnn_tester.cc b/paddle/fluid/inference/tests/api/analyzer_pyramid_dnn_tester.cc
index 9443b08063b8f61d3d6b291a7217d645d8825c54..1bb06aa21af55ce298756c1a95f4f95222218b6e 100644
--- a/paddle/fluid/inference/tests/api/analyzer_pyramid_dnn_tester.cc
+++ b/paddle/fluid/inference/tests/api/analyzer_pyramid_dnn_tester.cc
@@ -110,6 +110,11 @@ void SetConfig(AnalysisConfig *cfg) {
   if (FLAGS_zero_copy) {
     cfg->SwitchUseFeedFetchOps(false);
   }
+  // Enable runtime_context_cache_pass, disabled by default since it doesn't
+  // cover all the cases.
+  // See detail: https://github.com/PaddlePaddle/Paddle/issues/16609
+  // https://github.com/PaddlePaddle/Paddle/issues/16841
+  cfg->pass_builder()->AppendPass("runtime_context_cache_pass");
 }
 
 void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {