diff --git a/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h b/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h
index c5f55138d9bcf9c51c9c53c90d2ba465db76c594..e679f62a25ac2c2bc134e0e55bbed723b7fcd0bd 100644
--- a/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h
+++ b/paddle/fluid/operators/elementwise/mkldnn/elementwise_mkldnn_op.h
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #pragma once
+#include <string>
 #include <unordered_map>
 #include "paddle/fluid/memory/memcpy.h"
 #include "paddle/fluid/operators/elementwise/elementwise_add_op.h"
@@ -45,19 +46,25 @@ class EltwiseMKLDNNKernel : public framework::OpKernel<T> {
     float scale_x = ctx.Attr<float>("Scale_x");
     float scale_y = ctx.Attr<float>("Scale_y");
     float scale_o = ctx.Attr<float>("Scale_out");
-
     int axis = ctx.Attr<int>("axis");
 
+    bool is_inplaced = x->IsSharedBufferWith(*z);
+
+    std::string key = is_inplaced
+                          ? platform::CreateKey(dev_ctx, ctx.OutputName("Out"),
+                                                x->format(), y->format())
+                          : ctx.OutputName("Out");
+
     platform::BinaryMKLDNNHandler<T> handler(
         BINARY_OP, axis, dev_ctx, mkldnn_engine, ctx.GetPlace(), x, y, z,
-        scale_x, scale_y, scale_o, ctx.OutputName("Out"));
+        scale_x, scale_y, scale_o, key);
 
     const auto src_x_memory = handler.AcquireSrcMemory(x);
     const auto src_y_memory = handler.AcquireSecondSrcMemory(y);
 
     // For Inplace src and and dst are the same memory object
     const auto dst_memory =
-        x->IsSharedBufferWith(*z) ? src_x_memory : handler.AcquireDstMemory(z);
+        is_inplaced ? src_x_memory : handler.AcquireDstMemory(z);
 
     const auto binary_prim = handler.AcquireForwardPrimitive();