diff --git a/paddle/fluid/operators/sum_op.cc b/paddle/fluid/operators/sum_op.cc
index 71fcaafe6ba0e2267317f0d2aaa2e2853f9ed930..7abfbbd3cb5e5374441c511d82663788c39c04c6 100644
--- a/paddle/fluid/operators/sum_op.cc
+++ b/paddle/fluid/operators/sum_op.cc
@@ -52,10 +52,12 @@ class SumOp : public framework::OperatorWithKernel {
 
     framework::DDim in_dim({0});
     for (size_t i = 0; i < x_dims.size(); ++i) {
-      if (x_var_types[i] == framework::proto::VarType::SELECTED_ROWS) {
+      auto& x_dim = x_dims[i];
+      // x_dim.size() == 1 means the real dim of selected rows is [0]
+      if (x_var_types[i] == framework::proto::VarType::SELECTED_ROWS &&
+          x_dim.size() == 1) {
         continue;
       }
-      auto& x_dim = x_dims[i];
       if (framework::product(x_dim) == 0) {
         continue;
       }