diff --git a/paddle/fluid/operators/split_ids_op.h b/paddle/fluid/operators/split_ids_op.h
index d6caf626cb5613a69d0e58386006da5663f45432..c4af5a65fc5f81c1af7c1fdcca637ca37c940637 100644
--- a/paddle/fluid/operators/split_ids_op.h
+++ b/paddle/fluid/operators/split_ids_op.h
@@ -68,6 +68,9 @@ class SplitIdsOpKernel : public framework::OpKernel<T> {
       const auto &ids_rows = ids_selected_rows->rows();
       auto outs = ctx.MultiOutput<framework::SelectedRows>("Out");
       const size_t shard_num = outs.size();
+      for (auto &out : outs) {
+        out->mutable_rows()->clear();
+      }
       // get rows for outputs
       std::unordered_map<int64_t, size_t> id_to_index;
       for (size_t i = 0; i < ids_rows.size(); ++i) {