diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc
index 58b0d2a1189ad790d76d5cdc0b8f3eb9f48d2b28..0b39041c971e84b3ccc83312fca954eaf6e96946 100644
--- a/paddle/fluid/inference/api/analysis_config.cc
+++ b/paddle/fluid/inference/api/analysis_config.cc
@@ -948,8 +948,22 @@ void AnalysisConfig::Update() {
 #endif
   }
 
+  // TODO(inference): When we enable memory_optimize and mkldnn, PaddleSeg model
+  // fail.
   if (enable_memory_optim_) {
+#ifdef PADDLE_WITH_MKLDNN
+    if (use_mkldnn_) {
+      enable_memory_optim_ = false;
+      LOG_FIRST_N(WARNING, 1)
+          << "It is detected that mkldnn and memory_optimize_pass are enabled "
+             "at the same time, but they are not supported yet. Currently, "
+             "memory_optimize_pass is explicitly disabled";
+    } else {
+      pass_builder()->AppendAnalysisPass("memory_optimize_pass");
+    }
+#else
     pass_builder()->AppendAnalysisPass("memory_optimize_pass");
+#endif
   }
 
   if (use_lite_) {
diff --git a/paddle/fluid/inference/tests/api/analyzer_capi_exp_pd_config_tester.cc b/paddle/fluid/inference/tests/api/analyzer_capi_exp_pd_config_tester.cc
index cbf0e615c4d4eaacdf964e3078f835d8f50ef810..804ac21ea406c8b0755a875ce7be012394d7acf9 100644
--- a/paddle/fluid/inference/tests/api/analyzer_capi_exp_pd_config_tester.cc
+++ b/paddle/fluid/inference/tests/api/analyzer_capi_exp_pd_config_tester.cc
@@ -55,6 +55,10 @@ TEST(PD_Config, interface) {
   bool ir_optim = PD_ConfigIrOptim(config);
   EXPECT_TRUE(ir_optim);
 
+  PD_ConfigEnableMemoryOptim(config, true);
+  bool memory_enabled = PD_ConfigMemoryOptimEnabled(config);
+  EXPECT_TRUE(memory_enabled);
+
 #ifndef PADDLE_WITH_LITE
   PD_ConfigEnableLiteEngine(
       config, PD_PRECISION_FLOAT32, TRUE, 0, nullptr, 0, nullptr);
@@ -95,10 +99,6 @@ TEST(PD_Config, interface) {
   EXPECT_FALSE(onnxruntime_disabled);
   PD_ConfigEnableORTOptimization(config);
 
-  PD_ConfigEnableMemoryOptim(config, true);
-  bool memory_enabled = PD_ConfigMemoryOptimEnabled(config);
-  EXPECT_TRUE(memory_enabled);
-
   PD_ConfigEnableProfile(config);
   bool profile_enabled = PD_ConfigProfileEnabled(config);
   EXPECT_TRUE(profile_enabled);