diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc
index c7554de9df73b26f2b11a040940fbe91f557d612..1224caf88e6683687524a4a43e48e12946c3c441 100644
--- a/paddle/fluid/inference/api/analysis_config.cc
+++ b/paddle/fluid/inference/api/analysis_config.cc
@@ -948,12 +948,7 @@ void AnalysisConfig::Update() {
 #endif
   }
 
-#ifdef PADDLE_WITH_MKLDNN
-  // Do not optimize when mkldnn is on
-  if (enable_memory_optim_ && !use_mkldnn_) {
-#else
   if (enable_memory_optim_) {
-#endif
     pass_builder()->AppendAnalysisPass("memory_optimize_pass");
   }