diff --git a/paddle/fluid/operators/bmm_op.h b/paddle/fluid/operators/bmm_op.h
index 49104d4f08d288dd844545c0a8256fd22862ccad..15cd6de91365e0569aa97faa7462f7da3f97ebac 100644
--- a/paddle/fluid/operators/bmm_op.h
+++ b/paddle/fluid/operators/bmm_op.h
@@ -64,6 +64,10 @@ class BmmKernel : public framework::OpKernel<T> {
     Tensor *out = context.Output<Tensor>("Out");
     out->mutable_data<T>(context.GetPlace());
 
+    if (x.numel() == 0 || y.numel() == 0) {
+      return;
+    }
+
     auto blas = math::GetBlas<DeviceContext, T>(context);
 
     auto mat_dim_a = math::CreateMatrixDescriptor(x.dims(), 0, false);