diff --git a/paddle/fluid/framework/tensor_util.cc b/paddle/fluid/framework/tensor_util.cc
index e766c53713a371017341adae880a4c975724781d..8b2e3fc323ce560fbd5f8248584f95ef6a4bb702 100644
--- a/paddle/fluid/framework/tensor_util.cc
+++ b/paddle/fluid/framework/tensor_util.cc
@@ -48,19 +48,6 @@ void TensorCopy(const Tensor& src, const platform::Place& dst_place,
   }
 
   auto size = src.numel() * SizeOfType(src.type());
-#ifdef PADDLE_WITH_MKLDNN
-  if (src.layout() == DataLayout::kMKLDNN) {
-    PADDLE_ENFORCE_EQ(
-        src.memory_size(), dst->memory_size(),
-        platform::errors::InvalidArgument(
-            "When copying tensor with MKL-DNN data layout, "
-            "memory size of source tensor should be the same as memory size of "
-            "destination tensor. "
-            "But received src.memory_size = %d, dst.memory_size = %d.",
-            src.memory_size(), dst->memory_size()));
-    size = src.memory_size();
-  }
-#endif
 
   if (platform::is_cpu_place(src_place) && platform::is_cpu_place(dst_place)) {
     memory::Copy(boost::get<platform::CPUPlace>(dst_place), dst_ptr,
diff --git a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc
index 42b282ad27258af9d7176de540b09a9172b6d16d..5d05d089f5366765db90bf4372942ccf9d576a6b 100644
--- a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc
@@ -316,9 +316,11 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
         dst_memory_p = handler.AcquireDstMemoryFromResidualDataMemory(
             user_residual_memory_p, to_void_cast<T>(output_data), pipeline);
       } else {
-        auto output_data = output->mutable_data<T>(
-            ctx.GetPlace(), residual_param->memory_size());
-        framework::TensorCopy(*residual_param, residual_param->place(), output);
+        // Changing ShareDataWith to TensorCopy results in performance drop
+        // on ResNet architectures
+        // (https://github.com/PaddlePaddle/Paddle/issues/22964)
+        output->ShareDataWith(*residual_param);
+        auto output_data = output->mutable_data<T>(ctx.GetPlace());
         dst_memory_p =
             handler.AcquireDstMemoryFromPrimitive(to_void_cast<T>(output_data));
       }
@@ -611,8 +613,7 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
               ctx, output, residual_param, user_residual_md, handler,
               &pipeline);
         } else {
-          framework::TensorCopy(*residual_param, residual_param->place(),
-                                output);
+          output->ShareDataWith(*residual_param);
           dst_memory_p = platform::SetDstMemory<T_out>(ctx, output, handler);
         }
         need_s8_to_u8 =
@@ -683,7 +684,7 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
 
       if (fuse_residual_conn) {
         auto residual_param = ctx.Input<Tensor>("ResidualData");
-        framework::TensorCopy(*residual_param, residual_param->place(), output);
+        output->ShareDataWith(*residual_param);
         need_s8_to_u8 =
             (platform::MKLDNNGetDataType<T_out>() == memory::data_type::s8) &&
             unsigned_output;