diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc index 7b0650e962e59a0d37744a3d90b67883dfeae6d1..3a63ecb866fdbffbb335c09514e79bff7bd51b92 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc @@ -122,16 +122,6 @@ Status ClueOp::Reset() { return Status::OK(); } -Status ClueOp::LoadTensor(const std::string &line, std::unique_ptr *tensor_table, int64_t row) { - TensorRow tRow(1, nullptr); - (*tensor_table)->push_back(std::move(tRow)); - - std::shared_ptr tensor; - RETURN_IF_NOT_OK(Tensor::CreateScalar(line, &tensor)); - (**tensor_table)[row][0] = std::move(tensor); - return Status::OK(); -} - Status ClueOp::GetValue(const nlohmann::json &js, std::vector key_chain, std::shared_ptr *t) { nlohmann::json cursor = js; for (int i = 0; i < key_chain.size(); i++) { @@ -191,25 +181,24 @@ Status ClueOp::LoadFile(const std::string &file, const int64_t start_offset, con continue; } + nlohmann::json js; try { - nlohmann::json js = nlohmann::json::parse(line); - int cols_count = cols_to_keyword_.size(); - TensorRow tRow(cols_count, nullptr); - tensor_table->push_back(std::move(tRow)); - - int cout = 0; - for (auto &p : cols_to_keyword_) { - std::shared_ptr tensor; - RETURN_IF_NOT_OK(GetValue(js, p.second, &tensor)); - (*tensor_table)[rows_each_buffer][cout] = std::move(tensor); - cout++; - } + js = nlohmann::json::parse(line); } catch (const std::exception &err) { // Catch any exception and convert to Status return code RETURN_STATUS_UNEXPECTED("Failed to load json file"); } + int cols_count = cols_to_keyword_.size(); + TensorRow tRow(cols_count, nullptr); + tensor_table->push_back(std::move(tRow)); + int cout = 0; + for (auto &p : cols_to_keyword_) { + std::shared_ptr tensor; + RETURN_IF_NOT_OK(GetValue(js, p.second, &tensor)); + (*tensor_table)[rows_each_buffer][cout] = std::move(tensor); + cout++; + } - // RETURN_IF_NOT_OK(LoadTensor(line, &tensor_table, rows_each_buffer)); rows_each_buffer++; rows_total++; if (rows_each_buffer == rows_per_buffer_) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.h index da7f86b34e37c9acaf8216339af8cd12925fc7b9..eea6b5aa7ecdd39a0768a60ede744b894b04f006 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.h @@ -179,13 +179,6 @@ class ClueOp : public ParallelOp { // @return Status - the error code returned. Status WorkerEntry(int32_t worker_id) override; - // Parses a single row and puts the data into a tensor table. - // @param line - the content of the row. - // @param tensor_table - the tensor table to put the parsed data in. - // @param row - the id of the row filled in the tensor table. - // @return Status - the error code returned. - Status LoadTensor(const std::string &line, std::unique_ptr *tensor_table, int64_t row); - // Reads a clue file and loads the data into multiple buffers. // @param file - the file to read. // @param start_offset - the start offset of file.