diff --git a/paddle/fluid/framework/ir/trt_skip_layernorm_fuse_pass.cc b/paddle/fluid/framework/ir/trt_skip_layernorm_fuse_pass.cc
index 65afdec4c0564254eec02847f16923f2b24a3119..81f96f2fc33f455994c830c9307d89baf775742c 100644
--- a/paddle/fluid/framework/ir/trt_skip_layernorm_fuse_pass.cc
+++ b/paddle/fluid/framework/ir/trt_skip_layernorm_fuse_pass.cc
@@ -152,6 +152,13 @@ void TrtSkipLayerNormFusePass::ApplyImpl(ir::Graph *graph) const {
     }
 
     VLOG(4) << "handle TrtSkipLayerNorm fuse";
+
+    // x and y 's rank must be same
+    if (subgraph.at(x)->Var()->GetShape().size() !=
+        subgraph.at(y)->Var()->GetShape().size()) {
+      return;
+    }
+
     GET_IR_NODE_FROM_SUBGRAPH(elementwise, elementwise, fused_pattern);
     GET_IR_NODE_FROM_SUBGRAPH(elementwise_out, elementwise_out, fused_pattern);
     GET_IR_NODE_FROM_SUBGRAPH(layer_norm, layer_norm, fused_pattern);