diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc
index 612b926bd5385900e95bd1ea7879a97ad351b119..138675c1a48dc6518bd301b2761cfb34b73b0fd8 100644
--- a/paddle/fluid/inference/api/analysis_config.cc
+++ b/paddle/fluid/inference/api/analysis_config.cc
@@ -968,22 +968,8 @@ void AnalysisConfig::Update() {
 #endif
   }
 
-  // TODO(inference): When we enable memory_optimize and mkldnn, PaddleSeg model
-  // fail.
   if (enable_memory_optim_) {
-#ifdef PADDLE_WITH_MKLDNN
-    if (use_mkldnn_) {
-      enable_memory_optim_ = false;
-      LOG_FIRST_N(WARNING, 1)
-          << "It is detected that mkldnn and memory_optimize_pass are enabled "
-             "at the same time, but they are not supported yet. Currently, "
-             "memory_optimize_pass is explicitly disabled";
-    } else {
-      pass_builder()->AppendAnalysisPass("memory_optimize_pass");
-    }
-#else
     pass_builder()->AppendAnalysisPass("memory_optimize_pass");
-#endif
   }
 
   if (use_lite_) {
diff --git a/paddle/phi/kernels/onednn/conv_handler.h b/paddle/phi/kernels/onednn/conv_handler.h
index fd6cf0c577849bf91f137f35389f91fc6a611473..d41d1fdf0a0a9cd7d3c8ec13c869269f9a28a7b7 100644
--- a/paddle/phi/kernels/onednn/conv_handler.h
+++ b/paddle/phi/kernels/onednn/conv_handler.h
@@ -18,7 +18,9 @@
 #include "paddle/phi/backends/onednn/onednn_reuse.h"
 #include "paddle/phi/core/expect.h"
 #include "paddle/phi/core/macros.h"
+#include "paddle/phi/core/tensor_utils.h"
 #include "paddle/phi/kernels/cpu/conv_util.h"
+
 namespace phi {
 namespace onednn {
 
@@ -743,17 +745,9 @@ class ConvOneDNNHandlerT
   std::shared_ptr<dnnl::memory> AcquireDstMemoryWithResidual(
       phi::DenseTensor* output, const phi::DenseTensor* residual_param) {
     std::shared_ptr<dnnl::memory> dst_memory_p;
-    if (residual_param->mem_desc() != this->fwd_pd_->dst_desc()) {
-      auto residual_memory_p = this->AcquireResidualMemory(residual_param);
-      dst_memory_p = this->template AcquireDstMemory<T_out>(output);
-      this->AcquireReorder(residual_memory_p, dst_memory_p);
-    } else {
-      // Changing ShareDataWith to TensorCopy results in performance drop
-      // on ResNet architectures
-      // (https://github.com/PaddlePaddle/Paddle/issues/22964)
-      output->ShareDataWith(*residual_param);
-      dst_memory_p = this->template AcquireDstMemory<T_out>(output);
-    }
+    auto residual_memory_p = this->AcquireResidualMemory(residual_param);
+    dst_memory_p = this->template AcquireDstMemory<T_out>(output);
+    this->AcquireReorder(residual_memory_p, dst_memory_p);
     return dst_memory_p;
   }
 };