diff --git a/paddle/fluid/framework/ir/remove_padding_recover_padding_pass.cc b/paddle/fluid/framework/ir/remove_padding_recover_padding_pass.cc
index ffbff73183629284c72680039cb4459ffaf091de..b404d023d487b72fd7b7ca4ea763b3b4ab1361c5 100644
--- a/paddle/fluid/framework/ir/remove_padding_recover_padding_pass.cc
+++ b/paddle/fluid/framework/ir/remove_padding_recover_padding_pass.cc
@@ -453,7 +453,9 @@ void RemovePaddingRecoverPaddingPass::ApplyImpl(ir::Graph* graph) const {
     }
 
     if (PADDLE_GET_CONST(
-            int, matrix_multiply_op->Op()->GetAttr("x_num_col_dims")) != 2) {
+            int, matrix_multiply_op->Op()->GetAttr("x_num_col_dims")) != 2 &&
+        PADDLE_GET_CONST(
+            int, matrix_multiply_op->Op()->GetAttr("x_num_col_dims")) != -1) {
       check_flag = false;
     }
     if (!check_flag) {