diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt
index 0bbfe3c0e2ed638e9ec7457a84fcad7395f393e5..883575e41db2d883e9b969978419a10ffc58b97e 100644
--- a/paddle/fluid/framework/ir/CMakeLists.txt
+++ b/paddle/fluid/framework/ir/CMakeLists.txt
@@ -46,7 +46,6 @@ if(WITH_MKLDNN)
     pass_library(mkldnn_placement_pass base)
     pass_library(depthwise_conv_mkldnn_pass base)
     pass_library(conv_bias_mkldnn_fuse_pass inference)
-    pass_library(conv3d_bias_mkldnn_fuse_pass inference)
     pass_library(conv_relu_mkldnn_fuse_pass inference)
     pass_library(conv_elementwise_add_mkldnn_fuse_pass inference)
 endif()
diff --git a/paddle/fluid/framework/ir/conv3d_bias_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/conv3d_bias_mkldnn_fuse_pass.cc
deleted file mode 100644
index e2968ddf60d7740c4578e4474cd786121b33b575..0000000000000000000000000000000000000000
--- a/paddle/fluid/framework/ir/conv3d_bias_mkldnn_fuse_pass.cc
+++ /dev/null
@@ -1,18 +0,0 @@
-// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/fluid/framework/ir/conv3d_bias_mkldnn_fuse_pass.h"
-
-REGISTER_PASS(conv3d_bias_mkldnn_fuse_pass,
-              paddle::framework::ir::Conv3DBiasFusePass);
diff --git a/paddle/fluid/framework/ir/conv3d_bias_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/conv3d_bias_mkldnn_fuse_pass.h
deleted file mode 100644
index 5afe2cc61eebff802e739e0b6fdf9c53f5611465..0000000000000000000000000000000000000000
--- a/paddle/fluid/framework/ir/conv3d_bias_mkldnn_fuse_pass.h
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-#pragma once
-#include "paddle/fluid/framework/ir/conv_bias_mkldnn_fuse_pass.h"
-
-namespace paddle {
-namespace framework {
-namespace ir {
-/*
-* Fuse the Conv3D and Elementwise_add to a Conv3DBiasOp.
-*/
-class Conv3DBiasFusePass : public ConvBiasFusePass {
- public:
-  bool is_conv3d() const override { return true; }
-};
-}  // namespace ir
-}  // namespace framework
-}  // namespace paddle
diff --git a/paddle/fluid/framework/ir/conv_bias_mkldnn_fuse_pass.cc b/paddle/fluid/framework/ir/conv_bias_mkldnn_fuse_pass.cc
index c3ad3a0f183a564e06529a67519d900d7221b9ba..d4a701e0b173a96d8605dff308fee7007a0ecc0c 100644
--- a/paddle/fluid/framework/ir/conv_bias_mkldnn_fuse_pass.cc
+++ b/paddle/fluid/framework/ir/conv_bias_mkldnn_fuse_pass.cc
@@ -137,3 +137,5 @@ std::unique_ptr<ir::Graph> ConvBiasFusePass::ApplyImpl(
 }  // namespace paddle
 REGISTER_PASS(conv_bias_mkldnn_fuse_pass,
               paddle::framework::ir::ConvBiasFusePass);
+REGISTER_PASS(conv3d_bias_mkldnn_fuse_pass,
+              paddle::framework::ir::Conv3DBiasFusePass);
diff --git a/paddle/fluid/framework/ir/conv_bias_mkldnn_fuse_pass.h b/paddle/fluid/framework/ir/conv_bias_mkldnn_fuse_pass.h
index c3b58bf5822abf55f572ea159986a072430d1af3..f3ad9f1c2bf14db418629e0c607e2510f01908b8 100644
--- a/paddle/fluid/framework/ir/conv_bias_mkldnn_fuse_pass.h
+++ b/paddle/fluid/framework/ir/conv_bias_mkldnn_fuse_pass.h
@@ -32,6 +32,13 @@ class ConvBiasFusePass : public FusePassBase {
   std::unique_ptr<ir::Graph> ApplyImpl(std::unique_ptr<ir::Graph> graph) const;
   const std::string name_scope_{"conv_bias_mkldnn_fuse"};
 };
+/*
+* Fuse the Conv3D and Elementwise_add to a Conv3DBiasOp.
+*/
+class Conv3DBiasFusePass : public ConvBiasFusePass {
+ public:
+  bool is_conv3d() const override { return true; }
+};
 }  // namespace ir
 }  // namespace framework
 }  // namespace paddle
diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc
index ed99d9883b6da70ac42d585a95b7c98265ce4248..0118019df2f779a6409365555b530ae3b6d3971f 100644
--- a/paddle/fluid/framework/ir/graph_pattern_detector.cc
+++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc
@@ -1031,25 +1031,23 @@ PDNode *patterns::ElewiseAddActInplaceGrad::operator()(
 
 PDNode *patterns::ConvBias::operator()(
     paddle::framework::ir::PDNode *conv_input, bool is_conv3d) {
+  std::string type = is_conv3d ? "conv3d" : "conv2d";
   // Create Operators
-  conv_input->assert_is_op_input(is_conv3d ? "conv3d" : "conv2d", "Input");
-  auto *conv_op = pattern->NewNode(conv_repr())
-                      ->assert_is_op(is_conv3d ? "conv3d" : "conv2d");
+  conv_input->assert_is_op_input(type, "Input");
+  auto *conv_op = pattern->NewNode(conv_repr())->assert_is_op(type);
   auto *eltiwse_op =
       pattern->NewNode(eltwise_repr())->assert_is_op("elementwise_add");
   // Create variables
   // Filter
-  auto *conv_weight_var =
-      pattern->NewNode(conv_weight_repr())
-          ->AsInput()
-          ->assert_is_persistable_var()
-          ->assert_is_op_input(is_conv3d ? "conv3d" : "conv2d", "Filter");
+  auto *conv_weight_var = pattern->NewNode(conv_weight_repr())
+                              ->AsInput()
+                              ->assert_is_persistable_var()
+                              ->assert_is_op_input(type, "Filter");
   // intermediate variable, will be removed in the IR after fuse.
-  auto *conv_out_var =
-      pattern->NewNode(conv_out_repr())
-          ->AsIntermediate()
-          ->assert_is_only_output_of_op(is_conv3d ? "conv3d" : "conv2d")
-          ->assert_is_op_input("elementwise_add");
+  auto *conv_out_var = pattern->NewNode(conv_out_repr())
+                           ->AsIntermediate()
+                           ->assert_is_only_output_of_op(type)
+                           ->assert_is_op_input("elementwise_add");
   // Bias stored in elementwise_add
   auto *eltwise_bias_var = pattern->NewNode(eltwise_bias_repr())
                                ->AsInput()
diff --git a/paddle/fluid/operators/activation_mkldnn_op.cc b/paddle/fluid/operators/activation_mkldnn_op.cc
index 7fa81e185ebbab9655d29928ca523af0c535dceb..e16b6f78d16ce29cc493c4c795c7fe97a4bf2550 100644
--- a/paddle/fluid/operators/activation_mkldnn_op.cc
+++ b/paddle/fluid/operators/activation_mkldnn_op.cc
@@ -100,6 +100,10 @@ void eltwise_forward(const framework::ExecutionContext &ctx,
   const T *x_data = x->data<T>();
   T *y_data = y->mutable_data<T>(ctx.GetPlace());
 
+  PADDLE_ENFORCE(
+      x->dims().size() == 2 || x->dims().size() == 3 || x->dims().size() == 4,
+      "Input dim must be with 2, 3 or 4");
+
   std::vector<int> src_tz = framework::vectorize2int(x->dims());
 
   auto src_format =