diff --git a/paddle/fluid/framework/tensor_util.cc b/paddle/fluid/framework/tensor_util.cc
index 8b2e3fc323ce560fbd5f8248584f95ef6a4bb702..e766c53713a371017341adae880a4c975724781d 100644
--- a/paddle/fluid/framework/tensor_util.cc
+++ b/paddle/fluid/framework/tensor_util.cc
@@ -48,6 +48,19 @@ void TensorCopy(const Tensor& src, const platform::Place& dst_place,
   }
 
   auto size = src.numel() * SizeOfType(src.type());
+#ifdef PADDLE_WITH_MKLDNN
+  if (src.layout() == DataLayout::kMKLDNN) {
+    PADDLE_ENFORCE_EQ(
+        src.memory_size(), dst->memory_size(),
+        platform::errors::InvalidArgument(
+            "When copying tensor with MKL-DNN data layout, "
+            "memory size of source tensor should be the same as memory size of "
+            "destination tensor. "
+            "But received src.memory_size = %d, dst.memory_size = %d.",
+            src.memory_size(), dst->memory_size()));
+    size = src.memory_size();
+  }
+#endif
 
   if (platform::is_cpu_place(src_place) && platform::is_cpu_place(dst_place)) {
     memory::Copy(boost::get<platform::CPUPlace>(dst_place), dst_ptr,
diff --git a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc
index 7faf66a6e832acd17ef3b9788d4f8fedacd66d88..42b282ad27258af9d7176de540b09a9172b6d16d 100644
--- a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc
@@ -316,8 +316,9 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
         dst_memory_p = handler.AcquireDstMemoryFromResidualDataMemory(
             user_residual_memory_p, to_void_cast<T>(output_data), pipeline);
       } else {
-        output->ShareDataWith(*residual_param);
-        auto output_data = output->mutable_data<T>(ctx.GetPlace());
+        auto output_data = output->mutable_data<T>(
+            ctx.GetPlace(), residual_param->memory_size());
+        framework::TensorCopy(*residual_param, residual_param->place(), output);
         dst_memory_p =
             handler.AcquireDstMemoryFromPrimitive(to_void_cast<T>(output_data));
       }
@@ -610,7 +611,8 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
               ctx, output, residual_param, user_residual_md, handler,
               &pipeline);
         } else {
-          output->ShareDataWith(*residual_param);
+          framework::TensorCopy(*residual_param, residual_param->place(),
+                                output);
           dst_memory_p = platform::SetDstMemory<T_out>(ctx, output, handler);
         }
         need_s8_to_u8 =
@@ -681,7 +683,7 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
 
       if (fuse_residual_conn) {
         auto residual_param = ctx.Input<Tensor>("ResidualData");
-        output->ShareDataWith(*residual_param);
+        framework::TensorCopy(*residual_param, residual_param->place(), output);
         need_s8_to_u8 =
             (platform::MKLDNNGetDataType<T_out>() == memory::data_type::s8) &&
             unsigned_output;