diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake
index e4f25bc711d7c7145ed959561fc07ca0bad2b4a7..00a83a37893d9b9f91e37f8297393a27de98b2de 100644
--- a/cmake/external/mkldnn.cmake
+++ b/cmake/external/mkldnn.cmake
@@ -21,7 +21,7 @@ set(MKLDNN_INC_DIR
     "${MKLDNN_INSTALL_DIR}/include"
     CACHE PATH "mkldnn include directory." FORCE)
 set(MKLDNN_REPOSITORY ${GIT_URL}/oneapi-src/oneDNN.git)
-set(MKLDNN_TAG 9b186765dded79066e0cd9c17eb70b680b76fb8e)
+set(MKLDNN_TAG 2089770c4818be8933c5e9d1dd3cbaeba1457667)
 
 # Introduce variables:
 # * CMAKE_INSTALL_LIBDIR
diff --git a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc
index 0954414bee1906e990966aa9e200f5cb510c7c22..3fc9221260d5e467978cdec44278b11bdffd24d5 100644
--- a/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc
+++ b/paddle/fluid/framework/ir/mkldnn/softplus_activation_mkldnn_fuse_pass.cc
@@ -29,11 +29,6 @@ using string::PrettyLogDetail;
 void SoftplusActivationOneDNNPass::ApplyImpl(Graph *graph) const {
   auto act_types = GetSupportedActivations();
 
-  // Currently softplus can't be fused with hard_sigmoid
-  act_types.erase(
-      std::remove(act_types.begin(), act_types.end(), "hard_sigmoid"),
-      act_types.end());
-
   for (const auto &act_type : act_types) {
     FuseSoftplusActivation(graph, act_type);
   }
diff --git a/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
index 4c333565db32ac90bbcd29f4c5f93025269c39ce..50c22170623b2c2a27f10b5430aa0208dcd772c2 100644
--- a/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/fc_mkldnn_op.cc
@@ -171,42 +171,18 @@ class FCMKLDNNHandler
     const auto fuse_beta =
         ctx.HasAttr("fuse_beta") ? ctx.Attr<float>("fuse_beta") : 0.0f;
 
-    if (fuse_activation == "hard_sigmoid") {
-      post_ops.append_eltwise(activation_scale,
-                              dnnl::algorithm::eltwise_linear,
-                              fuse_alpha,
-                              fuse_beta);
-      post_ops.append_eltwise(
-          activation_scale, dnnl::algorithm::eltwise_clip, 0.0f, 1.0f);
-    } else {
-      const std::unordered_map<std::string, dnnl::algorithm> activation_map = {
-          {"abs", dnnl::algorithm::eltwise_abs},
-          {"clip", dnnl::algorithm::eltwise_clip},
-          {"gelu", dnnl::algorithm::eltwise_gelu_erf},
-          {"gelu_erf", dnnl::algorithm::eltwise_gelu_erf},
-          {"gelu_tanh", dnnl::algorithm::eltwise_gelu_tanh},
-          {"hard_swish", dnnl::algorithm::eltwise_hardswish},
-          {"leaky_relu", dnnl::algorithm::eltwise_relu},
-          {"mish", dnnl::algorithm::eltwise_mish},
-          {"relu", dnnl::algorithm::eltwise_relu},
-          {"relu6", dnnl::algorithm::eltwise_bounded_relu},
-          {"sigmoid", dnnl::algorithm::eltwise_logistic},
-          {"sqrt", dnnl::algorithm::eltwise_sqrt},
-          {"swish", dnnl::algorithm::eltwise_swish},
-          {"tanh", dnnl::algorithm::eltwise_tanh}};
-
-      const auto& activation_type = activation_map.find(fuse_activation);
-
-      PADDLE_ENFORCE_NE(
-          activation_type,
-          activation_map.end(),
-          platform::errors::InvalidArgument(
-              "Activation '%s' not found in oneDNN algorithms mapper",
-              fuse_activation));
-
-      post_ops.append_eltwise(
-          activation_scale, activation_type->second, fuse_alpha, fuse_beta);
-    }
+    const auto activation_map = phi::funcs::OneDNNActivationMap();
+    const auto& activation_type = activation_map.find(fuse_activation);
+
+    PADDLE_ENFORCE_NE(
+        activation_type,
+        activation_map.end(),
+        phi::errors::InvalidArgument(
+            "Activation '%s' not found in oneDNN algorithms mapper",
+            fuse_activation));
+
+    post_ops.append_eltwise(
+        activation_scale, activation_type->second, fuse_alpha, fuse_beta);
   }
 
   // Correct output scale, to take into account scaling of input and weights
diff --git a/paddle/fluid/operators/mkldnn/matmul_v2_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/matmul_v2_mkldnn_op.cc
index 7bf66bae93cc3e0659dce090415bac35c357e462..d7d3bd148e482ec310512c57cde77e6bdc504c48 100644
--- a/paddle/fluid/operators/mkldnn/matmul_v2_mkldnn_op.cc
+++ b/paddle/fluid/operators/mkldnn/matmul_v2_mkldnn_op.cc
@@ -173,59 +173,6 @@ class MatMulV2MKLDNNHandler
     this->AcquireForwardPrimitiveDescriptor(matmul_attrs, x_md, y_md, out_md);
   }
 
-  void AppendActivation(const ExecutionContext &ctx,
-                        dnnl::post_ops &post_ops,  // NOLINT
-                        float activation_scale = 1.0f) {
-    const auto invalid_attribute =
-        ctx.HasAttr("fuse_activation")
-            ? ctx.Attr<std::string>("fuse_activation").empty()
-            : true;
-    if (invalid_attribute) return;
-
-    const auto fuse_activation = ctx.Attr<std::string>("fuse_activation");
-    const auto fuse_alpha =
-        ctx.HasAttr("fuse_alpha") ? ctx.Attr<float>("fuse_alpha") : 0.0f;
-    const auto fuse_beta =
-        ctx.HasAttr("fuse_beta") ? ctx.Attr<float>("fuse_beta") : 0.0f;
-
-    if (fuse_activation == "hard_sigmoid") {
-      post_ops.append_eltwise(activation_scale,
-                              dnnl::algorithm::eltwise_linear,
-                              fuse_alpha,
-                              fuse_beta);
-      post_ops.append_eltwise(
-          activation_scale, dnnl::algorithm::eltwise_clip, 0.0f, 1.0f);
-    } else {
-      const std::unordered_map<std::string, dnnl::algorithm> activation_map = {
-          {"abs", dnnl::algorithm::eltwise_abs},
-          {"clip", dnnl::algorithm::eltwise_clip},
-          {"gelu", dnnl::algorithm::eltwise_gelu_erf},
-          {"gelu_erf", dnnl::algorithm::eltwise_gelu_erf},
-          {"gelu_tanh", dnnl::algorithm::eltwise_gelu_tanh},
-          {"hard_swish", dnnl::algorithm::eltwise_hardswish},
-          {"leaky_relu", dnnl::algorithm::eltwise_relu},
-          {"mish", dnnl::algorithm::eltwise_mish},
-          {"relu", dnnl::algorithm::eltwise_relu},
-          {"relu6", dnnl::algorithm::eltwise_bounded_relu},
-          {"sigmoid", dnnl::algorithm::eltwise_logistic},
-          {"sqrt", dnnl::algorithm::eltwise_sqrt},
-          {"swish", dnnl::algorithm::eltwise_swish},
-          {"tanh", dnnl::algorithm::eltwise_tanh}};
-
-      const auto &activation_type = activation_map.find(fuse_activation);
-
-      PADDLE_ENFORCE_NE(
-          activation_type,
-          activation_map.end(),
-          phi::errors::InvalidArgument(
-              "Activation '%s' not found in oneDNN algorithms mapper",
-              fuse_activation));
-
-      post_ops.append_eltwise(
-          activation_scale, activation_type->second, fuse_alpha, fuse_beta);
-    }
-  }
-
   float ComputeOutputScale(const ExecutionContext &ctx) {
     float alpha = ctx.HasAttr("alpha") ? ctx.Attr<float>("alpha") : 1.0f;
     if (ctx.HasAttr("Scale_x") && ctx.HasAttr("Scale_y") &&
@@ -264,8 +211,6 @@ class MatMulV2MKLDNNHandler
       }
     }
 
-    AppendActivation(ctx, post_operations);
-
     if (ctx.HasAttr("fused_output_scale")) {
       float scale_alpha = ctx.Attr<float>("fused_output_scale");
       post_operations.append_eltwise(
diff --git a/paddle/phi/backends/onednn/onednn_reuse.h b/paddle/phi/backends/onednn/onednn_reuse.h
index 3c8a7292d9886b820f51408afb9c72efdc082152..9b5c068decfbb33daba4b5da16b16a74d0fa1c2a 100644
--- a/paddle/phi/backends/onednn/onednn_reuse.h
+++ b/paddle/phi/backends/onednn/onednn_reuse.h
@@ -50,6 +50,24 @@ constexpr bool is_bfloat16() {
   return std::is_same<T, dtype::bfloat16>::value;
 }
 
+static std::unordered_map<std::string, dnnl::algorithm> OneDNNActivationMap() {
+  return {{"abs", dnnl::algorithm::eltwise_abs},
+          {"clip", dnnl::algorithm::eltwise_clip},
+          {"gelu", dnnl::algorithm::eltwise_gelu_erf},
+          {"gelu_erf", dnnl::algorithm::eltwise_gelu_erf},
+          {"gelu_tanh", dnnl::algorithm::eltwise_gelu_tanh},
+          {"hard_sigmoid", dnnl::algorithm::eltwise_hardsigmoid},
+          {"hard_swish", dnnl::algorithm::eltwise_hardswish},
+          {"leaky_relu", dnnl::algorithm::eltwise_relu},
+          {"mish", dnnl::algorithm::eltwise_mish},
+          {"relu", dnnl::algorithm::eltwise_relu},
+          {"relu6", dnnl::algorithm::eltwise_bounded_relu},
+          {"sigmoid", dnnl::algorithm::eltwise_logistic},
+          {"sqrt", dnnl::algorithm::eltwise_sqrt},
+          {"swish", dnnl::algorithm::eltwise_swish},
+          {"tanh", dnnl::algorithm::eltwise_tanh}};
+}
+
 static void AppendActivation(const OneDNNContext& dev_ctx,
                              dnnl::post_ops& post_ops,  // NOLINT
                              float activation_scale = 1.0f,
@@ -78,42 +96,18 @@ static void AppendActivation(const OneDNNContext& dev_ctx,
                     : 0.0f;
   }
 
-  if (fuse_activation == "hard_sigmoid") {
-    post_ops.append_eltwise(activation_scale,
-                            dnnl::algorithm::eltwise_linear,
-                            fuse_alpha,
-                            fuse_beta);
-    post_ops.append_eltwise(
-        activation_scale, dnnl::algorithm::eltwise_clip, 0.0f, 1.0f);
-  } else {
-    const std::unordered_map<std::string, dnnl::algorithm> activation_map = {
-        {"abs", dnnl::algorithm::eltwise_abs},
-        {"clip", dnnl::algorithm::eltwise_clip},
-        {"gelu", dnnl::algorithm::eltwise_gelu_erf},
-        {"gelu_erf", dnnl::algorithm::eltwise_gelu_erf},
-        {"gelu_tanh", dnnl::algorithm::eltwise_gelu_tanh},
-        {"hard_swish", dnnl::algorithm::eltwise_hardswish},
-        {"leaky_relu", dnnl::algorithm::eltwise_relu},
-        {"mish", dnnl::algorithm::eltwise_mish},
-        {"relu", dnnl::algorithm::eltwise_relu},
-        {"relu6", dnnl::algorithm::eltwise_bounded_relu},
-        {"sigmoid", dnnl::algorithm::eltwise_logistic},
-        {"sqrt", dnnl::algorithm::eltwise_sqrt},
-        {"swish", dnnl::algorithm::eltwise_swish},
-        {"tanh", dnnl::algorithm::eltwise_tanh}};
-
-    const auto& activation_type = activation_map.find(fuse_activation);
-
-    PADDLE_ENFORCE_NE(
-        activation_type,
-        activation_map.end(),
-        errors::InvalidArgument(
-            "Activation '%s' not found in oneDNN algorithms mapper",
-            fuse_activation));
+  const auto activation_map = OneDNNActivationMap();
 
-    post_ops.append_eltwise(
-        activation_scale, activation_type->second, fuse_alpha, fuse_beta);
-  }
+  const auto& activation_type = activation_map.find(fuse_activation);
+
+  PADDLE_ENFORCE_NE(activation_type,
+                    activation_map.end(),
+                    errors::InvalidArgument(
+                        "Activation '%s' not found in oneDNN algorithms mapper",
+                        fuse_activation));
+
+  post_ops.append_eltwise(
+      activation_scale, activation_type->second, fuse_alpha, fuse_beta);
 }
 
 template <typename T,
diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_onednn_softplus_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_onednn_softplus_activation_fuse_pass.py
index 46c28653949baa56bf9bd2540acb5593ff797c81..c36149114b948603a5d34d7c82fff78c2411e8cb 100644
--- a/python/paddle/fluid/tests/unittests/ir/inference/test_onednn_softplus_activation_fuse_pass.py
+++ b/python/paddle/fluid/tests/unittests/ir/inference/test_onednn_softplus_activation_fuse_pass.py
@@ -33,6 +33,7 @@ class TestSoftplusActivationOneDNNFusePass(PassAutoScanTest):
                     'swish',
                     'mish',
                     'sqrt',
+                    'hard_sigmoid',
                     'hard_swish',
                     'abs',
                     'relu6',