提交 3d12d368 编写于 作者: M mindspore-ci-bot 提交者: Gitee

!3332 Fix csv dataset shard

Merge pull request !3332 from jiangzhiwen/fix_csv_shard
...@@ -130,6 +130,7 @@ int CsvOp::CsvParser::put_row(char c) { ...@@ -130,6 +130,7 @@ int CsvOp::CsvParser::put_row(char c) {
} }
if (total_rows_ >= end_offset_) { if (total_rows_ >= end_offset_) {
cur_col_ = 0;
return 0; return 0;
} }
...@@ -353,7 +354,9 @@ Status CsvOp::CsvParser::initCsvParser() { ...@@ -353,7 +354,9 @@ Status CsvOp::CsvParser::initCsvParser() {
{{State::END_OF_LINE, Message::MS_NORMAL}, {{State::END_OF_LINE, Message::MS_NORMAL},
{State::UNQUOTE, {State::UNQUOTE,
[this](CsvParser &, char c) -> int { [this](CsvParser &, char c) -> int {
this->tensor_table_->push_back(TensorRow(column_default_.size(), nullptr)); if (this->total_rows_ > this->start_offset_ && this->total_rows_ <= this->end_offset_) {
this->tensor_table_->push_back(TensorRow(column_default_.size(), nullptr));
}
this->str_buf_[0] = c; this->str_buf_[0] = c;
this->pos_ = 1; this->pos_ = 1;
return 0; return 0;
...@@ -361,14 +364,18 @@ Status CsvOp::CsvParser::initCsvParser() { ...@@ -361,14 +364,18 @@ Status CsvOp::CsvParser::initCsvParser() {
{{State::END_OF_LINE, Message::MS_DELIM}, {{State::END_OF_LINE, Message::MS_DELIM},
{State::DELIM, {State::DELIM,
[this](CsvParser &, char c) -> int { [this](CsvParser &, char c) -> int {
this->tensor_table_->push_back(TensorRow(column_default_.size(), nullptr)); if (this->total_rows_ > this->start_offset_ && this->total_rows_ <= this->end_offset_) {
this->tensor_table_->push_back(TensorRow(column_default_.size(), nullptr));
}
this->put_record(c); this->put_record(c);
return 0; return 0;
}}}, }}},
{{State::END_OF_LINE, Message::MS_QUOTE}, {{State::END_OF_LINE, Message::MS_QUOTE},
{State::QUOTE, {State::QUOTE,
[this](CsvParser &, char c) -> int { [this](CsvParser &, char c) -> int {
this->tensor_table_->push_back(TensorRow(column_default_.size(), nullptr)); if (this->total_rows_ > this->start_offset_ && this->total_rows_ <= this->end_offset_) {
this->tensor_table_->push_back(TensorRow(column_default_.size(), nullptr));
}
return 0; return 0;
}}}, }}},
{{State::END_OF_LINE, Message::MS_END_OF_LINE}, {State::END_OF_LINE, &CsvParser::null_func}}, {{State::END_OF_LINE, Message::MS_END_OF_LINE}, {State::END_OF_LINE, &CsvParser::null_func}},
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册