diff --git a/paddle/fluid/framework/executor.cc b/paddle/fluid/framework/executor.cc
index 44646f7ab1ae3ef823c8d12bd6283b042df5a814..14dd821367e376430703277fa690a359861669f2 100644
--- a/paddle/fluid/framework/executor.cc
+++ b/paddle/fluid/framework/executor.cc
@@ -93,6 +93,20 @@ ExecutorPrepareContext::~ExecutorPrepareContext() {
 
 Executor::Executor(const platform::Place& place) : place_(place) {}
 
+Executor::~Executor() {
+#ifdef PADDLE_WITH_MKLDNN
+  // Clear mkl-dnn cache, unless explicitly
+  // (as set in constructor) marked not to do so
+  // this is needed to have mkl-dnn unit tests working
+  if (platform::is_cpu_place(place_)) {
+    platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
+    platform::MKLDNNDeviceContext* dev_ctx =
+        (platform::MKLDNNDeviceContext*)pool.Get(place_);
+    dev_ctx->ResetBlobMap();
+  }
+#endif
+}
+
 void Executor::Close() {
 #ifdef PADDLE_WITH_DISTRIBUTE
   // TODO(typhoonzero): complete message will need to use real trainer_id,
diff --git a/paddle/fluid/framework/executor.h b/paddle/fluid/framework/executor.h
index 587ac1a8a6f694bdceea691704da368b72dededb..6ca50b70031db7649f444c5de5f88e72d6828614 100644
--- a/paddle/fluid/framework/executor.h
+++ b/paddle/fluid/framework/executor.h
@@ -58,6 +58,7 @@ class Executor {
 
   explicit Executor(const platform::Place& place);
 
+  ~Executor();
   /*
    * Close this Executor.
    * Calling this method will send complete messages to all pserver instances.
diff --git a/paddle/fluid/operators/elementwise/mkldnn/elementwise_add_mkldnn_op.cc b/paddle/fluid/operators/elementwise/mkldnn/elementwise_add_mkldnn_op.cc
index 1f4a4fb0e16756ad0c8234d4e15e03278b6ff21c..a3a5a031e971338ba59c88c4bd303f60fee55889 100644
--- a/paddle/fluid/operators/elementwise/mkldnn/elementwise_add_mkldnn_op.cc
+++ b/paddle/fluid/operators/elementwise/mkldnn/elementwise_add_mkldnn_op.cc
@@ -136,9 +136,8 @@ class EltwiseAddMKLDNNKernel : public framework::OpKernel<T> {
       std::vector<memory::primitive_desc> srcs_pd;
       std::vector<float> scales = {1.0f, 1.0f};
 
-      const std::string key = platform::CreateKey(
-          src_x_tz, ctx.op().Output("Out") + std::to_string(x->format()) +
-                        std::to_string(y->format()));
+      const std::string key =
+          platform::CreateKey(src_x_tz, ctx.op().Output("Out"));
 
       platform::SumMKLDNNHandler handler(dev_ctx, mkldnn_engine, key);
 
diff --git a/paddle/fluid/operators/mkldnn/batch_norm_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/batch_norm_mkldnn_op.cc
index 6b1c870c3c148e169c32a7c04134a9134845079c..189f512d31694ecba3021319039a75049d659335 100644
--- a/paddle/fluid/operators/mkldnn/batch_norm_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/batch_norm_mkldnn_op.cc
@@ -40,8 +40,7 @@ class BatchNormMKLDNNHandler
       : platform::MKLDNNHandlerT<T, mkldnn::batch_normalization_forward,
                                  mkldnn::batch_normalization_backward>(
             dev_ctx, dev_ctx.GetEngine(), cpu_place,
-            platform::CreateKey(dims, epsilon, flags, global_stats, fmt,
-                                uniq_name)) {
+            platform::CreateKey(dims, uniq_name)) {
     auto md = mkldnn::memory::desc(dims, platform::MKLDNNGetDataType<T>(), fmt);
 
     this->AcquireForwardPrimitiveDescriptor(
@@ -59,8 +58,7 @@ class BatchNormMKLDNNHandler
       : platform::MKLDNNHandlerT<T, mkldnn::batch_normalization_forward,
                                  mkldnn::batch_normalization_backward>(
             dev_ctx, dev_ctx.GetEngine(), cpu_place,
-            platform::CreateKey(dims, epsilon, flags, false, src_fmt,
-                                uniq_name)) {
+            platform::CreateKey(dims, uniq_name)) {
     auto diff_dst_md =
         mkldnn::memory::desc(dims, platform::MKLDNNGetDataType<T>(), diff_fmt);
     auto src_md =
diff --git a/paddle/fluid/operators/mkldnn/concat_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/concat_mkldnn_op.cc
index 8010b52a1dbfc3f10ef479322e08995ae598d8f3..4baf65fb7426eddfeb4bc70fde1124c31c77ed90 100644
--- a/paddle/fluid/operators/mkldnn/concat_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/concat_mkldnn_op.cc
@@ -150,9 +150,8 @@ class ConcatMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
 
     ConcatPrimitiveFactory<T> prim_creator;
     std::string key = platform::CreateKey(
-        paddle::framework::vectorize<int>(multi_input[0]->dims()), concat_axis,
-        ctx.op().Output("Out"), dt, multi_input[0]->format(),
-        platform::ThreadIDasStr());
+        paddle::framework::vectorize<int>(multi_input[0]->dims()),
+        ctx.op().Output("Out"), dt, platform::ThreadIDasStr());
     const std::string key_prim = key + "@concat_p";
     const std::string key_concat_pd = key + "@concat_pd";
     const std::string key_srcs = key + "@concat_srcs";
diff --git a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc
index 86c7d7a5cc624c37519245c0db47eeaeffed6375..a9fc17ce89b89a8b56ce6043221c968cd387c896 100644
--- a/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/conv_mkldnn_op.cc
@@ -220,8 +220,7 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
 
     // Get unique name for storing MKLDNN primitives
     const std::string key = platform::CreateKey(
-        src_tz, weights_tz, fuse_activation, strides, paddings, dilations,
-        groups, ctx.op().Input("Input") + ctx.op().Input("Filter"));
+        src_tz, ctx.op().Input("Input") + ctx.op().Input("Filter"));
 
     std::vector<primitive> pipeline;
 
@@ -450,9 +449,7 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
         paddle::framework::ToMKLDNNDataType(input->type());
 
     std::string key = platform::CreateKey(
-        src_tz, weights_tz, strides, paddings, dilations, groups, src_dt,
-        input->format(), fuse_activation, fuse_residual_conn,
-        ctx.op().Input("Input") + ctx.op().Input("Filter"));
+        src_tz, src_dt, ctx.op().Input("Input") + ctx.op().Input("Filter"));
 
     std::shared_ptr<mkldnn::convolution_forward> conv_p;
     std::shared_ptr<mkldnn::memory> src_memory_p;
@@ -662,8 +659,7 @@ class ConvMKLDNNGradOpKernel : public paddle::framework::OpKernel<T> {
     // as well as attributes of primitive to be created
     // This name will be used as key when saving info into device context
     const std::string key = platform::CreateKey(
-        src_tz, weights_tz, "", strides, paddings, dilations, groups,
-        ctx.op().Input("Input") + ctx.op().Input("Filter"));
+        src_tz, ctx.op().Input("Input") + ctx.op().Input("Filter"));
 
     const std::string key_conv_pd = key + "@conv_pd";
     std::vector<primitive> pipeline;
diff --git a/paddle/fluid/operators/mkldnn/conv_transpose_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/conv_transpose_mkldnn_op.cc
index 84240d30fe1d1aa6d1e04b2c103d22887b79f4ca..2af20228c81de4c68523e54bcb9d87eef00e590f 100644
--- a/paddle/fluid/operators/mkldnn/conv_transpose_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/conv_transpose_mkldnn_op.cc
@@ -128,8 +128,7 @@ class ConvTransposeMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
 
     // Get unique name for storing MKLDNN primitives
     const std::string key =
-        platform::CreateKey(src_tz, weights_tz, strides, paddings, dilations,
-                            groups, ctx.op().Output("Output"));
+        platform::CreateKey(src_tz, ctx.op().Output("Output"));
 
     std::vector<mkldnn::primitive> pipeline;
 
diff --git a/paddle/fluid/operators/mkldnn/dequantize_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/dequantize_mkldnn_op.cc
index b74e7127ea3314695f866e0c2f2693cc3489b145..4353c62136507872a6f34267f03b0f9386432112 100644
--- a/paddle/fluid/operators/mkldnn/dequantize_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/dequantize_mkldnn_op.cc
@@ -52,8 +52,8 @@ class DeQuantOpKernel : public framework::OpKernel<T> {
     mkldnn::memory::data_type src_dt =
         paddle::framework::ToMKLDNNDataType(input->type());
     MKLDNNMemoryFormat src_fmt = input->format();
-    std::string key = platform::CreateKey(src_dt, src_tz, reorder_scale[0],
-                                          ctx.op().Output("Output"));
+    std::string key =
+        platform::CreateKey(src_dt, src_tz, ctx.op().Output("Output"));
     const std::string key_prim = key + "@reorder_p";
     const std::string key_src_mem = key + "@src_mem";
     const std::string key_dst_mem = key + "@dst_mem";
diff --git a/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
index a910deef528e2672a2553121557df8cbab4dae49..349dbffb386f84662b0ce8e6668126263d95b88d 100644
--- a/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
@@ -228,8 +228,7 @@ std::shared_ptr<FCPrimitiveFactory<T>> GetPrimitiveFactory(
     const Tensor* input, const Tensor* weights,
     const mkldnn::engine& mkldnn_engine) {
   const std::string key = platform::CreateKey(
-      input->format(), framework::vectorize<int>(weights->dims()),
-      ctx.op().Output("Out"));
+      framework::vectorize<int>(weights->dims()), ctx.op().Output("Out"));
 
   auto prim_creator =
       std::static_pointer_cast<FCPrimitiveFactory<T>>(dev_ctx.GetBlob(key));
diff --git a/paddle/fluid/operators/mkldnn/mul_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/mul_mkldnn_op.cc
index 4bdd93d08ecf98336c292a58a9b6de7235e475e2..7d41960214d42b7592e866e8d31cb5aa0271f90f 100644
--- a/paddle/fluid/operators/mkldnn/mul_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/mul_mkldnn_op.cc
@@ -340,8 +340,7 @@ std::shared_ptr<MulPrimitiveFactory<XT, YT, OT>> GetPrimitiveFactory(
     const Tensor *input_x, const Tensor *input_y,
     const mkldnn::engine &mkldnn_engine, bool enable_quant) {
   const std::string key = platform::CreateKey(
-      input_x->format(), input_x->type(),
-      framework::vectorize<int>(input_x->dims()), input_y->format(),
+      input_x->type(), framework::vectorize<int>(input_x->dims()),
       input_y->type(), framework::vectorize<int>(input_y->dims()),
       ctx.op().Output("Out"));
 
diff --git a/paddle/fluid/operators/mkldnn/softmax_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/softmax_mkldnn_op.cc
index 690f9271fb7cb17032ef56d4904855a0ec115e6a..f6b0a9ac528e57d2f131fa2e974ff1d0dbb620a0 100644
--- a/paddle/fluid/operators/mkldnn/softmax_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/softmax_mkldnn_op.cc
@@ -45,7 +45,7 @@ class SoftmaxMKLDNNHandler
       : platform::MKLDNNHandlerT<T, mkldnn::softmax_forward,
                                  mkldnn::softmax_backward>(
             dev_ctx, dev_ctx.GetEngine(), cpu_place,
-            platform::CreateKey(dims, axis, uniq_name)) {
+            platform::CreateKey(dims, uniq_name)) {
     auto md = mkldnn::memory::desc(dims, platform::MKLDNNGetDataType<T>(), fmt);
 
     this->AcquireForwardPrimitiveDescriptor(prop_kind::forward_scoring, md,
@@ -60,7 +60,7 @@ class SoftmaxMKLDNNHandler
       : platform::MKLDNNHandlerT<T, mkldnn::softmax_forward,
                                  mkldnn::softmax_backward>(
             dev_ctx, dev_ctx.GetEngine(), cpu_place,
-            platform::CreateKey(dims, axis, uniq_name)) {
+            platform::CreateKey(dims, uniq_name)) {
     auto data_softmax_md =
         mkldnn::memory::desc(dims, platform::MKLDNNGetDataType<T>(), fmt);
     auto diff_softmax_md =
diff --git a/paddle/fluid/operators/mkldnn/transpose_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/transpose_mkldnn_op.cc
index bcf919fadcf5e164136c57f3cd640bea288bda1f..a091122c5c178c252b7934f9457edd65d64ea749 100644
--- a/paddle/fluid/operators/mkldnn/transpose_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/transpose_mkldnn_op.cc
@@ -46,8 +46,7 @@ class TransposeMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
     auto nchw_tz = paddle::framework::vectorize<int>(input->dims());
 
     const std::string key =
-        platform::CreateKey(nchw_tz, axis, ctx.op().Output("Out") +
-                                               std::to_string(input->format()));
+        platform::CreateKey(nchw_tz, ctx.op().Output("Out"));
 
     platform::TransposeMKLDNNHandler handler(nchw_tz, axis, dev_ctx,
                                              mkldnn_engine, key);
@@ -100,7 +99,7 @@ class TransposeMKLDNNGradOpKernel : public paddle::framework::OpKernel<T> {
     auto nchw_tz = paddle::framework::vectorize<int>(out_grad->dims());
 
     const std::string key = platform::CreateKey(
-        nchw_tz, axis, ctx.op().Output(framework::GradVarName("X")));
+        nchw_tz, ctx.op().Output(framework::GradVarName("X")));
 
     platform::TransposeMKLDNNHandler handler(nchw_tz, reversed_axis, dev_ctx,
                                              mkldnn_engine, key);
diff --git a/paddle/fluid/platform/mkldnn_reuse.h b/paddle/fluid/platform/mkldnn_reuse.h
index 50c629e53ef280975346fac9a3912c0b6ea6c924..aa0b3d7d3359c4f31312c0ef539786d10ce444f0 100644
--- a/paddle/fluid/platform/mkldnn_reuse.h
+++ b/paddle/fluid/platform/mkldnn_reuse.h
@@ -420,8 +420,7 @@ class ActivationMKLDNNHandler
       : platform::MKLDNNHandlerT<T, mkldnn::eltwise_forward,
                                  mkldnn::eltwise_backward>(
             dev_ctx, dev_ctx.GetEngine(), cpu_place,
-            platform::CreateKey(dims, algorithm, fmt, alpha, beta,
-                                unique_name)) {
+            platform::CreateKey(dims, unique_name)) {
     auto md = mkldnn::memory::desc(dims, platform::MKLDNNGetDataType<T>(), fmt);
 
     this->AcquireForwardPrimitiveDescriptor(
@@ -441,8 +440,7 @@ class ActivationMKLDNNHandler
       : platform::MKLDNNHandlerT<T, mkldnn::eltwise_forward,
                                  mkldnn::eltwise_backward>(
             dev_ctx, dev_ctx.GetEngine(), cpu_place,
-            platform::CreateKey(dims, algorithm, fmt, alpha, beta,
-                                unique_name)) {
+            platform::CreateKey(dims, unique_name)) {
     auto diff_dst_md = platform::MKLDNNMemDesc(
         dims, platform::MKLDNNGetDataType<T>(), diff_fmt);
     auto src_md =
@@ -473,7 +471,7 @@ class LRNMKLDNNHandler
 
       : platform::MKLDNNHandlerT<T, mkldnn::lrn_forward, mkldnn::lrn_backward>(
             dev_ctx, dev_ctx.GetEngine(), cpu_place,
-            platform::CreateKey(dims, n, alpha, beta, k, fmt, unique_name)) {
+            platform::CreateKey(dims, unique_name)) {
     auto src_md =
         mkldnn::memory::desc(dims, platform::MKLDNNGetDataType<T>(), fmt);
     this->AcquireForwardPrimitiveDescriptor(
@@ -491,7 +489,7 @@ class LRNMKLDNNHandler
 
       : platform::MKLDNNHandlerT<T, mkldnn::lrn_forward, mkldnn::lrn_backward>(
             dev_ctx, dev_ctx.GetEngine(), cpu_place,
-            platform::CreateKey(dims, n, alpha, beta, k, fmt, unique_name)) {
+            platform::CreateKey(dims, unique_name)) {
     auto src_md =
         mkldnn::memory::desc(dims, platform::MKLDNNGetDataType<T>(), fmt);
     auto diff_md =
@@ -533,8 +531,7 @@ class PoolingMKLDNNHandler : public MKLDNNHandlerT<T, mkldnn::pooling_forward,
       : platform::MKLDNNHandlerT<T, mkldnn::pooling_forward,
                                  mkldnn::pooling_backward>(
             dev_ctx, dev_ctx.GetEngine(), cpu_place,
-            platform::CreateKey(src_dims, pooling_type, ksize, strides,
-                                paddings, dt, fmt, unique_name)) {
+            platform::CreateKey(src_dims, dt, unique_name)) {
     auto src_md = mkldnn::memory::desc(src_dims, dt, fmt);
     /* create memory descriptor for pooling without specified format
      * ('any') which lets a primitive (pooling in this case) choose
@@ -574,8 +571,7 @@ class PoolingMKLDNNHandler : public MKLDNNHandlerT<T, mkldnn::pooling_forward,
       : platform::MKLDNNHandlerT<T, mkldnn::pooling_forward,
                                  mkldnn::pooling_backward>(
             dev_ctx, dev_ctx.GetEngine(), cpu_place,
-            platform::CreateKey(diff_src_dims, pooling_type, ksize, strides,
-                                paddings, dt, fmt, unique_name)) {
+            platform::CreateKey(diff_src_dims, dt, unique_name)) {
     auto diff_dst_md = mkldnn::memory::desc(
         diff_dst_dims, platform::MKLDNNGetDataType<T>(), diff_dst_fmt);
     auto diff_src_md =