From 0c5e47111b1241cdd71f0eb9f087a15a6fe29bc3 Mon Sep 17 00:00:00 2001 From: hesham Date: Thu, 11 Jun 2020 00:34:22 -0400 Subject: [PATCH] - Move MutableBuffer to be protected - Add new CreateTensors --- mindspore/ccsrc/dataset/core/data_type.h | 59 +++++++++-------- mindspore/ccsrc/dataset/core/tensor.cc | 45 +++++++++++++ mindspore/ccsrc/dataset/core/tensor.h | 43 ++++++++++-- .../engine/datasetops/device_queue_op.cc | 2 +- .../engine/datasetops/source/celeba_op.cc | 14 +--- .../engine/datasetops/source/cifar_op.cc | 9 ++- .../engine/datasetops/source/coco_op.cc | 13 +--- .../datasetops/source/image_folder_op.cc | 14 +--- .../engine/datasetops/source/manifest_op.cc | 18 +---- .../engine/datasetops/source/mnist_op.cc | 2 +- .../source/sampler/distributed_sampler.cc | 4 +- .../datasetops/source/sampler/pk_sampler.cc | 4 +- .../source/sampler/random_sampler.cc | 2 +- .../source/sampler/sequential_sampler.cc | 2 +- .../source/sampler/subset_random_sampler.cc | 2 +- .../source/sampler/weighted_random_sampler.cc | 2 +- .../engine/datasetops/source/tf_reader_op.cc | 48 +------------- .../engine/datasetops/source/tf_reader_op.h | 13 +--- .../engine/datasetops/source/voc_op.cc | 12 +--- mindspore/ccsrc/dataset/engine/gnn/graph.cc | 2 +- .../ccsrc/dataset/engine/tdt/tdt_plugin.cc | 2 +- .../ccsrc/dataset/kernels/data/data_utils.h | 4 +- .../dataset/kernels/image/image_utils.cc | 13 ++-- .../ccsrc/dataset/kernels/image/image_utils.h | 2 +- .../image/random_crop_decode_resize_op.cc | 4 +- tests/ut/cpp/dataset/common/cvop_common.cc | 14 +--- tests/ut/cpp/dataset/image_folder_op_test.cc | 8 +-- tests/ut/cpp/dataset/map_op_test.cc | 8 +-- .../random_crop_decode_resize_op_test.cc | 15 +++-- .../cpp/dataset/stand_alone_samplers_test.cc | 8 +-- tests/ut/cpp/dataset/tensor_string_test.cc | 8 +-- tests/ut/cpp/dataset/tensor_test.cc | 66 ++++++++++--------- 32 files changed, 212 insertions(+), 250 deletions(-) diff --git a/mindspore/ccsrc/dataset/core/data_type.h b/mindspore/ccsrc/dataset/core/data_type.h index f1f0bb2eb..a487f3300 100644 --- a/mindspore/ccsrc/dataset/core/data_type.h +++ b/mindspore/ccsrc/dataset/core/data_type.h @@ -128,7 +128,9 @@ class DataType { // @tparam T // @return true or false template - bool IsCompatible() const; + bool IsCompatible() const { + return type_ == FromCType(); + } // returns true if the template type is the same as the Tensor type_ // @tparam T @@ -146,6 +148,9 @@ class DataType { return out; } + template + static DataType FromCType(); + // Convert from DataType to Pybind type // @return py::dtype AsNumpyType() const; @@ -191,68 +196,68 @@ class DataType { }; template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_BOOL; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_BOOL); } template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_FLOAT64; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_FLOAT64); } template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_FLOAT32; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_FLOAT32); } template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_FLOAT16; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_FLOAT16); } template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_INT64; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_INT64); } template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_UINT64; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_UINT64); } template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_INT32; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_INT32); } template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_UINT32; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_UINT32); } template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_INT16; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_INT16); } template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_UINT16; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_UINT16); } template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_INT8; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_INT8); } template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_UINT8; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_UINT8); } template <> -inline bool DataType::IsCompatible() const { - return type_ == DataType::DE_STRING; +inline DataType DataType::FromCType() { + return DataType(DataType::DE_STRING); } template <> diff --git a/mindspore/ccsrc/dataset/core/tensor.cc b/mindspore/ccsrc/dataset/core/tensor.cc index aa7205853..539c7adab 100644 --- a/mindspore/ccsrc/dataset/core/tensor.cc +++ b/mindspore/ccsrc/dataset/core/tensor.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -311,6 +312,50 @@ Status Tensor::CreateTensor(std::shared_ptr *ptr, const dataengine::Byte return Status::OK(); } +Status Tensor::CreateTensor(std::shared_ptr *ptr, const std::string &file_path) { + std::ifstream fs; + fs.open(file_path, std::ios::binary | std::ios::in); + CHECK_FAIL_RETURN_UNEXPECTED(!fs.fail(), "Fail to open file: " + file_path); + int64_t num_bytes = fs.seekg(0, std::ios::end).tellg(); + CHECK_FAIL_RETURN_UNEXPECTED(fs.seekg(0, std::ios::beg).good(), "Fail to find size of file"); + RETURN_IF_NOT_OK( + Tensor::CreateTensor(ptr, TensorImpl::kFlexible, TensorShape{num_bytes}, DataType(DataType::DE_UINT8))); + int64_t written_bytes = fs.read(reinterpret_cast((*ptr)->GetMutableBuffer()), num_bytes).gcount(); + CHECK_FAIL_RETURN_UNEXPECTED(written_bytes == num_bytes && fs.good(), "Error in writing to tensor"); + fs.close(); + return Status::OK(); +} + +Status Tensor::CreateTensor(std::shared_ptr *ptr, const dataengine::BytesList &bytes_list, + const TensorShape &shape, const DataType &type, dsize_t pad_size) { + RETURN_IF_NOT_OK(Tensor::CreateTensor(ptr, TensorImpl::kFlexible, shape, type)); + + unsigned char *current_tensor_addr = (*ptr)->GetMutableBuffer(); + int64_t tensor_bytes_remaining = bytes_list.value_size() * pad_size; + + for (int i = 0; i < bytes_list.value_size(); i++) { + // read string data into tensor + const std::string ¤t_element = bytes_list.value(i); + int return_code = + memcpy_s(current_tensor_addr, tensor_bytes_remaining, common::SafeCStr(current_element), current_element.size()); + + CHECK_FAIL_RETURN_UNEXPECTED(return_code == 0, "memcpy_s failed when reading bytesList element into Tensor"); + + current_tensor_addr += current_element.size(); + tensor_bytes_remaining -= current_element.size(); + + // pad + int64_t chars_to_pad = pad_size - current_element.size(); + return_code = memset_s(current_tensor_addr, tensor_bytes_remaining, static_cast(' '), chars_to_pad); + CHECK_FAIL_RETURN_UNEXPECTED(return_code == 0, "memcpy_s failed when padding Tensor"); + + current_tensor_addr += chars_to_pad; + tensor_bytes_remaining -= chars_to_pad; + } + + return Status::OK(); +} + // Memcpy the given strided array's used part to consecutive memory // Consider a 3-d array // A[(i * shape[1] + j) * shape[2] + k] = B[i][j][k] = C[i * strides[0] + j * strides[1] + k * strides[2]] diff --git a/mindspore/ccsrc/dataset/core/tensor.h b/mindspore/ccsrc/dataset/core/tensor.h index 1d176004d..406156c32 100644 --- a/mindspore/ccsrc/dataset/core/tensor.h +++ b/mindspore/ccsrc/dataset/core/tensor.h @@ -135,9 +135,41 @@ class Tensor { static Status CreateTensor(std::shared_ptr *ptr, const std::vector &strings, const TensorShape &shape = TensorShape::CreateUnknownRankShape()); + // create tensor from protobuf bytelist with strings static Status CreateTensor(std::shared_ptr *ptr, const dataengine::BytesList &bytes_list, const TensorShape &shape); + // A static factory method to create a Tensor from a given list of numbers. + // @param ptr output argument to hold the created Tensor + // @param items elements of the tensor + // @param shape shape of the tensor + // @return Status Code + template + static Status CreateTensor(std::shared_ptr *ptr, const std::vector &items, + const TensorShape &shape_req = TensorShape::CreateUnknownRankShape()) { + DataType type = DataType::FromCType(); + auto items_ptr = reinterpret_cast(&items[0]); + TensorShape shape = shape_req; + if (!shape.known()) { + shape = TensorShape({static_cast(items.size())}); + } + return CreateTensor(ptr, TensorImpl::kFlexible, shape, type, items_ptr); + } + + // A static factory method to create a Tensor from a given number. + // @param ptr output argument to hold the created Tensor + // @param item value + // @return Status Code + template + static Status CreateTensor(std::shared_ptr *ptr, const T &item) { + return CreateTensor(ptr, {item}, TensorShape::CreateScalar()); + } + // Create tensor from protobuf bytelist with uint8 or int8 types + static Status CreateTensor(std::shared_ptr *ptr, const dataengine::BytesList &bytes_list, + const TensorShape &shape, const DataType &type, dsize_t pad_size); + + static Status CreateTensor(std::shared_ptr *ptr, const std::string &path); + // Copy raw data of a array based on shape and strides to the destination pointer // @param dst Pointer to the destination array where the content is to be copied // @param src Pointer to the source of strided array to be copied @@ -260,11 +292,6 @@ class Tensor { // @return const unsigned char* const unsigned char *GetBuffer() const; - // Get the starting memory address for the data of the tensor. This potentially - // drives an allocation if the data area. - // @return unsigned char* - unsigned char *GetMutableBuffer(); - // Getter of the type // @return DataType type() const { return type_; } @@ -518,6 +545,7 @@ class Tensor { // @return TensorIterator template TensorIterator begin() { + AllocateBuffer(SizeInBytes()); return TensorIterator(data_); } @@ -536,6 +564,11 @@ class Tensor { Status CopyLastDimAt(const std::shared_ptr &src, const std::vector &index); protected: + // Get the starting memory address for the data of the tensor. This potentially + // drives an allocation if the data is null. + // @return unsigned char* + unsigned char *GetMutableBuffer(); + // A function that prints Tensor recursively, first called by print // @param out // @param cur_dim diff --git a/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc index a2b55ecfb..18bf65951 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc @@ -254,7 +254,7 @@ Status DeviceQueueOp::MallocForGPUData(std::vector *items, return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "memory malloc failed."); } (void)memset_s(sub_item.data_ptr_, sub_item.data_len_, 0, sub_item.data_len_); - unsigned char *column_data = curr_row[i]->GetMutableBuffer(); + const unsigned char *column_data = curr_row[i]->GetBuffer(); if (memcpy_s(sub_item.data_ptr_, sub_item.data_len_, column_data, static_cast(curr_row[i++]->SizeInBytes())) != 0) { MS_LOG(ERROR) << "memcpy_s failed!"; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc index 69fda9b66..e293db3ba 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc @@ -363,19 +363,7 @@ Status CelebAOp::LoadTensorRow(const std::pair Path path(folder_path_); Path image_path = path / image_label.first; - std::ifstream handle(image_path.toString(), std::ios::binary | std::ios::in); - if (handle.fail()) { - std::string err_msg = "Fail to open file: " + image_path.toString(); - return Status(StatusCode::kFileNotExist, __LINE__, __FILE__, err_msg); - } - - (void)handle.seekg(0, std::ios::end); - int64_t num_elements = handle.tellg(); - (void)handle.seekg(0, std::ios::beg); - RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data_schema_->column(0).tensorImpl(), - TensorShape(std::vector(1, num_elements)), - data_schema_->column(0).type())); - (void)handle.read(reinterpret_cast(image->GetMutableBuffer()), num_elements); + RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, image_path.toString())); if (decode_ == true) { Status rc = Decode(image, &image); if (rc.IsError()) { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc index 0893ad9f8..296b52e2c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc @@ -195,7 +195,7 @@ Status CifarOp::LoadTensorRow(uint64_t index, TensorRow *trow) { std::shared_ptr fine_label; std::shared_ptr ori_image = cifar_image_label_pairs_[index].first; std::shared_ptr copy_image = - std::make_shared(ori_image->shape(), ori_image->type(), ori_image->GetMutableBuffer()); + std::make_shared(ori_image->shape(), ori_image->type(), ori_image->GetBuffer()); RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(), data_schema_->column(1).shape(), data_schema_->column(1).type(), reinterpret_cast(&cifar_image_label_pairs_[index].second[0]))); @@ -366,10 +366,9 @@ Status CifarOp::ParseCifarData() { RETURN_IF_NOT_OK(Tensor::CreateTensor(&image_tensor, data_schema_->column(0).tensorImpl(), TensorShape({kCifarImageHeight, kCifarImageWidth, kCifarImageChannel}), data_schema_->column(0).type())); - for (int ch = 0; ch < kCifarImageChannel; ++ch) { - for (int pix = 0; pix < kCifarImageHeight * kCifarImageWidth; ++pix) { - (image_tensor->GetMutableBuffer())[pix * kCifarImageChannel + ch] = block[cur_block_index++]; - } + auto itr = image_tensor->begin(); + for (; itr != image_tensor->end(); itr++) { + *itr = block[cur_block_index++]; } cifar_image_label_pairs_.emplace_back(std::make_pair(image_tensor, labels)); } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.cc index 9f49f887e..15a37bd7b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.cc @@ -591,17 +591,8 @@ Status CocoOp::LaunchThreadsAndInitOp() { } Status CocoOp::ReadImageToTensor(const std::string &path, const ColDescriptor &col, std::shared_ptr *tensor) { - std::ifstream fs; - fs.open(path, std::ios::binary | std::ios::in); - if (fs.fail()) { - RETURN_STATUS_UNEXPECTED("Fail to open file: " + path); - } - int64_t num_elements = fs.seekg(0, std::ios::end).tellg(); - (void)fs.seekg(0, std::ios::beg); - RETURN_IF_NOT_OK( - Tensor::CreateTensor(tensor, col.tensorImpl(), TensorShape(std::vector(1, num_elements)), col.type())); - (void)fs.read(reinterpret_cast((*tensor)->GetMutableBuffer()), num_elements); - fs.close(); + RETURN_IF_NOT_OK(Tensor::CreateTensor(tensor, path)); + if (decode_ == true) { Status rc = Decode(*tensor, tensor); if (rc.IsError()) { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc index bd7da566b..7b2c1b6a0 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc @@ -204,18 +204,8 @@ Status ImageFolderOp::LoadTensorRow(ImageLabelPair pairPtr, TensorRow *trow) { RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(), data_schema_->column(1).shape(), data_schema_->column(1).type(), reinterpret_cast(&pairPtr->second))); - std::ifstream fs; - fs.open(folder_path_ + (pairPtr->first), std::ios::binary | std::ios::in); - if (fs.fail()) { - RETURN_STATUS_UNEXPECTED("Fail to open file: " + pairPtr->first); - } - int64_t num_elements = fs.seekg(0, std::ios::end).tellg(); - (void)fs.seekg(0, std::ios::beg); - RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data_schema_->column(0).tensorImpl(), - TensorShape(std::vector(1, num_elements)), - data_schema_->column(0).type(), nullptr)); - (void)fs.read(reinterpret_cast(image->GetMutableBuffer()), num_elements); - fs.close(); + RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, folder_path_ + (pairPtr->first))); + if (decode_ == true) { Status rc = Decode(image, &image); if (rc.IsError()) { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc index 14cc9b22d..7532c2385 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc @@ -198,23 +198,7 @@ Status ManifestOp::LoadTensorRow(const std::paircolumn(1).type(), reinterpret_cast(&label_index[0]))); } - std::ifstream fs; - fs.open(data.first, std::ios::binary | std::ios::in); - if (!fs.is_open()) { - RETURN_STATUS_UNEXPECTED("Fail to open file: " + data.first); - } - - int64_t num_elements = fs.seekg(0, std::ios::end).tellg(); - (void)fs.seekg(0, std::ios::beg); - RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data_schema_->column(0).tensorImpl(), - TensorShape(std::vector(1, num_elements)), - data_schema_->column(0).type(), nullptr)); - (void)fs.read(reinterpret_cast(image->GetMutableBuffer()), num_elements); - if (fs.fail()) { - fs.close(); - RETURN_STATUS_UNEXPECTED("Fail to read file: " + data.first); - } - fs.close(); + RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data.first)); if (decode_ == true) { Status rc = Decode(image, &image); if (rc.IsError()) { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc index a937d8e5a..5018f93c5 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc @@ -167,7 +167,7 @@ Status MnistOp::LoadTensorRow(const MnistLabelPair &mnist_pair, TensorRow *trow) int32_t l = mnist_pair.second; // make a copy of cached tensor RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data_schema_->column(0).tensorImpl(), mnist_pair.first->shape(), - mnist_pair.first->type(), mnist_pair.first->GetMutableBuffer())); + mnist_pair.first->type(), mnist_pair.first->GetBuffer())); RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(), data_schema_->column(1).shape(), data_schema_->column(1).type(), reinterpret_cast(&l))); (*trow) = {std::move(image), std::move(label)}; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc index e1f3ed721..61a53aa89 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc @@ -68,8 +68,8 @@ Status DistributedSampler::GetNextSample(std::unique_ptr *out_buffer (*out_buffer) = std::make_unique(cnt_, DataBuffer::kDeBFlagNone); std::shared_ptr sample_ids; RETURN_IF_NOT_OK(CreateSamplerTensor(&sample_ids, samples_per_buffer_)); - int64_t *id_ptr = reinterpret_cast(sample_ids->GetMutableBuffer()); - while (cnt_ < samples_per_buffer_) { + auto id_ptr = sample_ids->begin(); + while (cnt_ < samples_per_buffer_ && id_ptr != sample_ids->end()) { int64_t sampled_id = (num_devices_ * cnt_ + device_id_) % num_rows_; if (shuffle_) { sampled_id = shuffle_vec_[static_cast(sampled_id)]; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc index 0c49c1b31..6dd771bbd 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc @@ -73,8 +73,8 @@ Status PKSampler::GetNextSample(std::unique_ptr *out_buffer) { std::shared_ptr sample_ids; int64_t last_id = (samples_per_buffer_ + next_id_ > num_samples_) ? num_samples_ : samples_per_buffer_ + next_id_; RETURN_IF_NOT_OK(CreateSamplerTensor(&sample_ids, last_id - next_id_)); - int64_t *id_ptr = reinterpret_cast(sample_ids->GetMutableBuffer()); - while (next_id_ < last_id) { + auto id_ptr = sample_ids->begin(); + while (next_id_ < last_id && id_ptr != sample_ids->end()) { int64_t cls_id = next_id_++ / samples_per_class_; const std::vector &samples = label_to_ids_[labels_[cls_id]]; int64_t rnd_ind = std::uniform_int_distribution(0, samples.size() - 1)(rnd_); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc index c78225e01..b051c0cbf 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc @@ -45,7 +45,7 @@ Status RandomSampler::GetNextSample(std::unique_ptr *out_buffer) { std::shared_ptr sampleIds; int64_t last_id = std::min(samples_per_buffer_ + next_id_, num_samples_); RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, last_id - next_id_)); - int64_t *id_ptr = reinterpret_cast(sampleIds->GetMutableBuffer()); + auto id_ptr = sampleIds->begin(); for (int64_t i = 0; i < (last_id - next_id_); i++) { int64_t sampled_id = 0; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc index 8a088ef9c..1119d701e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc @@ -42,7 +42,7 @@ Status SequentialSampler::GetNextSample(std::unique_ptr *out_buffer) int64_t num_elements = std::min(remaining_ids, samples_per_buffer_); RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, num_elements)); - int64_t *idPtr = reinterpret_cast(sampleIds->GetMutableBuffer()); + auto idPtr = sampleIds->begin(); for (int64_t i = 0; i < num_elements; i++) { int64_t sampled_id = current_id_; if (HasChildSampler()) { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc index 21eead3a0..15ce91168 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc @@ -94,7 +94,7 @@ Status SubsetRandomSampler::GetNextSample(std::unique_ptr *out_buffe RETURN_IF_NOT_OK(CreateSamplerTensor(&outputIds, last_id - sample_id_)); // Initialize tensor - int64_t *id_ptr = reinterpret_cast(outputIds->GetMutableBuffer()); + auto id_ptr = outputIds->begin(); while (sample_id_ < last_id) { if (indices_[sample_id_] >= num_rows_) { std::string err_msg = diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc index cdf728cbc..c3277cbf4 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc @@ -125,7 +125,7 @@ Status WeightedRandomSampler::GetNextSample(std::unique_ptr *out_buf RETURN_IF_NOT_OK(CreateSamplerTensor(&outputIds, last_id - sample_id_)); // Initialize tensor. - int64_t *id_ptr = reinterpret_cast(outputIds->GetMutableBuffer()); + auto id_ptr = outputIds->begin(); // Assign the data to tensor element. while (sample_id_ < last_id) { int64_t genId; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc index e347ee470..208965e88 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc @@ -771,53 +771,7 @@ Status TFReaderOp::LoadBytesList(const ColDescriptor ¤t_col, const dataeng // know how many elements there are and the total bytes, create tensor here: TensorShape current_shape = TensorShape::CreateScalar(); RETURN_IF_NOT_OK(current_col.MaterializeTensorShape((*num_elements) * pad_size, ¤t_shape)); - RETURN_IF_NOT_OK(Tensor::CreateTensor(tensor, current_col.tensorImpl(), current_shape, current_col.type())); - - // Tensors are lazily allocated, this eagerly allocates memory for the tensor. - unsigned char *current_tensor_addr = (*tensor)->GetMutableBuffer(); - int64_t tensor_bytes_remaining = (*num_elements) * pad_size; - - if (current_tensor_addr == nullptr) { - std::string err_msg = "tensor memory allocation failed"; - RETURN_STATUS_UNEXPECTED(err_msg); - } - - RETURN_IF_NOT_OK(LoadAndPadBytes(current_tensor_addr, bytes_list, tensor_bytes_remaining, pad_size)); - - return Status::OK(); -} - -Status TFReaderOp::LoadAndPadBytes(unsigned char *current_tensor_addr, const dataengine::BytesList &bytes_list, - int64_t tensor_bytes_remaining, int64_t pad_size) { - if (current_tensor_addr == nullptr) { - std::string err_msg = "current_tensor_addr is null"; - RETURN_STATUS_UNEXPECTED(err_msg); - } - - for (int i = 0; i < bytes_list.value_size(); i++) { - // read string data into tensor - const std::string ¤t_element = bytes_list.value(i); - int return_code = - memcpy_s(current_tensor_addr, tensor_bytes_remaining, common::SafeCStr(current_element), current_element.size()); - if (return_code != 0) { - std::string err_msg = "memcpy_s failed when reading bytesList element into Tensor"; - RETURN_STATUS_UNEXPECTED(err_msg); - } - - current_tensor_addr += current_element.size(); - tensor_bytes_remaining -= current_element.size(); - - // pad - int64_t chars_to_pad = pad_size - current_element.size(); - return_code = memset_s(current_tensor_addr, tensor_bytes_remaining, static_cast(' '), chars_to_pad); - if (return_code != 0) { - std::string err_msg = "memset_s failed when padding bytesList in Tensor"; - RETURN_STATUS_UNEXPECTED(err_msg); - } - - current_tensor_addr += chars_to_pad; - tensor_bytes_remaining -= chars_to_pad; - } + RETURN_IF_NOT_OK(Tensor::CreateTensor(tensor, bytes_list, current_shape, current_col.type(), pad_size)); return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h index 17728136c..79131c026 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h @@ -296,17 +296,8 @@ class TFReaderOp : public ParallelOp { // @param column_values_list - the cell that contains the bytes list to read from. // @param elementStr - the string we read the value into. // @return Status - the error code returned. - Status LoadBytesList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, - int32_t *num_elements, std::shared_ptr *tensor); - - // Loads all the strings in bytes_list into the memory at current_tensor_addr. - // @param current_tensor_addr - the memory address to load the strings to. - // @param bytes_list - the list of strings to load. - // @param tensor_bytes_remaining - the number of bytes available for this function to use. - // @param pad_size - number of bytes to pad to. - // @return Status - the error code returned. - Status LoadAndPadBytes(unsigned char *current_tensor_addr, const dataengine::BytesList &bytes_list, - int64_t tensor_bytes_remaining, int64_t pad_size); + static Status LoadBytesList(const ColDescriptor ¤t_col, const dataengine::Feature &column_values_list, + int32_t *num_elements, std::shared_ptr *tensor); // Reads values from a float list // @param current_col - the column descriptor containing the expected shape and type of the data. diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc index 3d52fc737..c2b4d05d3 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc @@ -368,17 +368,7 @@ Status VOCOp::LaunchThreadsAndInitOp() { } Status VOCOp::ReadImageToTensor(const std::string &path, const ColDescriptor &col, std::shared_ptr *tensor) { - std::ifstream fs; - fs.open(path, std::ios::binary | std::ios::in); - if (fs.fail()) { - RETURN_STATUS_UNEXPECTED("Fail to open file: " + path); - } - int64_t num_elements = fs.seekg(0, std::ios::end).tellg(); - (void)fs.seekg(0, std::ios::beg); - RETURN_IF_NOT_OK( - Tensor::CreateTensor(tensor, col.tensorImpl(), TensorShape(std::vector(1, num_elements)), col.type())); - (void)fs.read(reinterpret_cast((*tensor)->GetMutableBuffer()), num_elements); - fs.close(); + RETURN_IF_NOT_OK(Tensor::CreateTensor(tensor, path)); if (decode_ == true) { Status rc = Decode(*tensor, tensor); if (rc.IsError()) { diff --git a/mindspore/ccsrc/dataset/engine/gnn/graph.cc b/mindspore/ccsrc/dataset/engine/gnn/graph.cc index 2ac3f3f5b..15222d3b2 100644 --- a/mindspore/ccsrc/dataset/engine/gnn/graph.cc +++ b/mindspore/ccsrc/dataset/engine/gnn/graph.cc @@ -59,7 +59,7 @@ Status Graph::CreateTensorByVector(const std::vector> &data, Data size_t n = data[0].size(); RETURN_IF_NOT_OK(Tensor::CreateTensor( &tensor, TensorImpl::kFlexible, TensorShape({static_cast(m), static_cast(n)}), type, nullptr)); - T *ptr = reinterpret_cast(tensor->GetMutableBuffer()); + auto ptr = tensor->begin(); for (const auto &id_m : data) { CHECK_FAIL_RETURN_UNEXPECTED(id_m.size() == n, "Each member of the vector has a different size"); for (const auto &id_n : id_m) { diff --git a/mindspore/ccsrc/dataset/engine/tdt/tdt_plugin.cc b/mindspore/ccsrc/dataset/engine/tdt/tdt_plugin.cc index 51fff6bbd..383460cb0 100644 --- a/mindspore/ccsrc/dataset/engine/tdt/tdt_plugin.cc +++ b/mindspore/ccsrc/dataset/engine/tdt/tdt_plugin.cc @@ -119,7 +119,7 @@ TdtStatus TdtPlugin::translate(const TensorRow &ts_row, std::vector &i data_item.tensorShape_ = dataShapes; data_item.tensorType_ = datatype; data_item.dataLen_ = ts->SizeInBytes(); - data_item.dataPtr_ = std::shared_ptr(reinterpret_cast(ts->GetMutableBuffer()), [](void *elem) {}); + data_item.dataPtr_ = std::shared_ptr(reinterpret_cast(&(*ts->begin())), [](void *elem) {}); items.emplace_back(data_item); MS_LOG(INFO) << "TDT data type is " << datatype << ", data shape is " << dataShapes << ", data length is " << ts->Size() << "."; diff --git a/mindspore/ccsrc/dataset/kernels/data/data_utils.h b/mindspore/ccsrc/dataset/kernels/data/data_utils.h index a0eb15d6d..bfd514122 100644 --- a/mindspore/ccsrc/dataset/kernels/data/data_utils.h +++ b/mindspore/ccsrc/dataset/kernels/data/data_utils.h @@ -101,11 +101,11 @@ Status PadEndString(const std::shared_ptr &src, std::shared_ptr const std::vector &pad_shape, const std::string &pad_val); // recursive helper function for padding string tensors. This function could be very expensive if called on a -// multi-dimensional tensor it is only meant to be called by PadEndNumeric. +// multi-dimensional tensor it is only meant to be called by PadEndString. // @tparam T - type of tensor and fill value // @param std::shared_ptr src - Tensor to pad from // @param std::shared_ptr* dst - Tensor to pad to, return value -// @param std::vector cur_ind - recursion helper +// @param std::vector cur_ind - recursion helperas text // @param std::string pad_val - value to pad tensor with // @param size_t cur_dim - recursion helper // @return Status - The error code return diff --git a/mindspore/ccsrc/dataset/kernels/image/image_utils.cc b/mindspore/ccsrc/dataset/kernels/image/image_utils.cc index 3e4708f49..7c31a94b5 100644 --- a/mindspore/ccsrc/dataset/kernels/image/image_utils.cc +++ b/mindspore/ccsrc/dataset/kernels/image/image_utils.cc @@ -119,17 +119,14 @@ Status Resize(const std::shared_ptr &input, std::shared_ptr *out } } -bool HasJpegMagic(const unsigned char *data, size_t data_size) { +bool HasJpegMagic(const std::shared_ptr &input) { const unsigned char *kJpegMagic = (unsigned char *)"\xFF\xD8\xFF"; constexpr size_t kJpegMagicLen = 3; - return data_size >= kJpegMagicLen && memcmp(data, kJpegMagic, kJpegMagicLen) == 0; + return input->SizeInBytes() >= kJpegMagicLen && memcmp(input->GetBuffer(), kJpegMagic, kJpegMagicLen) == 0; } Status Decode(const std::shared_ptr &input, std::shared_ptr *output) { - if (input->GetMutableBuffer() == nullptr) { - RETURN_STATUS_UNEXPECTED("Tensor is nullptr"); - } - if (HasJpegMagic(input->GetMutableBuffer(), input->SizeInBytes())) { + if (HasJpegMagic(input)) { return JpegCropAndDecode(input, output); } else { return DecodeCv(input, output); @@ -283,7 +280,7 @@ Status JpegCropAndDecode(const std::shared_ptr &input, std::shared_ptrGetMutableBuffer(), input->SizeInBytes()); + JpegSetSource(&cinfo, input->GetBuffer(), input->SizeInBytes()); (void)jpeg_read_header(&cinfo, TRUE); RETURN_IF_NOT_OK(JpegSetColorSpace(&cinfo)); jpeg_calc_output_dimensions(&cinfo); @@ -312,7 +309,7 @@ Status JpegCropAndDecode(const std::shared_ptr &input, std::shared_ptr(ts, DataType(DataType::DE_UINT8)); const int buffer_size = output_tensor->SizeInBytes(); - JSAMPLE *buffer = static_cast(output_tensor->GetMutableBuffer()); + JSAMPLE *buffer = static_cast(reinterpret_cast(&(*output_tensor->begin()))); const int max_scanlines_to_read = skipped_scanlines + crop_h; // stride refers to output tensor, which has 3 components at most const int stride = crop_w * kOutNumComponents; diff --git a/mindspore/ccsrc/dataset/kernels/image/image_utils.h b/mindspore/ccsrc/dataset/kernels/image/image_utils.h index 394323974..b44f953f9 100644 --- a/mindspore/ccsrc/dataset/kernels/image/image_utils.h +++ b/mindspore/ccsrc/dataset/kernels/image/image_utils.h @@ -96,7 +96,7 @@ Status Decode(const std::shared_ptr &input, std::shared_ptr *out Status DecodeCv(const std::shared_ptr &input, std::shared_ptr *output); -bool HasJpegMagic(const unsigned char *data, size_t data_size); +bool HasJpegMagic(const std::shared_ptr &input); void JpegSetSource(j_decompress_ptr c_info, const void *data, int64_t data_size); diff --git a/mindspore/ccsrc/dataset/kernels/image/random_crop_decode_resize_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_crop_decode_resize_op.cc index c11b5b596..74aa91ea7 100644 --- a/mindspore/ccsrc/dataset/kernels/image/random_crop_decode_resize_op.cc +++ b/mindspore/ccsrc/dataset/kernels/image/random_crop_decode_resize_op.cc @@ -31,7 +31,7 @@ Status RandomCropDecodeResizeOp::Compute(const std::shared_ptr &input, s if (input == nullptr) { RETURN_STATUS_UNEXPECTED("input tensor is null"); } - if (!HasJpegMagic(input->GetMutableBuffer(), input->SizeInBytes())) { + if (!HasJpegMagic(input)) { DecodeOp op(true); std::shared_ptr decoded; RETURN_IF_NOT_OK(op.Compute(input, &decoded)); @@ -43,7 +43,7 @@ Status RandomCropDecodeResizeOp::Compute(const std::shared_ptr &input, s jerr.pub.error_exit = JpegErrorExitCustom; try { jpeg_create_decompress(&cinfo); - JpegSetSource(&cinfo, input->GetMutableBuffer(), input->SizeInBytes()); + JpegSetSource(&cinfo, input->GetBuffer(), input->SizeInBytes()); (void)jpeg_read_header(&cinfo, TRUE); jpeg_calc_output_dimensions(&cinfo); } catch (std::runtime_error &e) { diff --git a/tests/ut/cpp/dataset/common/cvop_common.cc b/tests/ut/cpp/dataset/common/cvop_common.cc index 7ee080dd6..6f66229e8 100644 --- a/tests/ut/cpp/dataset/common/cvop_common.cc +++ b/tests/ut/cpp/dataset/common/cvop_common.cc @@ -28,9 +28,9 @@ namespace common = mindspore::common; using namespace mindspore::dataset; -using mindspore::MsLogLevel::INFO; -using mindspore::ExceptionType::NoExceptionType; using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::INFO; using UT::CVOP::CVOpCommon; CVOpCommon::CVOpCommon() {} @@ -52,15 +52,7 @@ std::string CVOpCommon::GetFilename() { void CVOpCommon::GetInputImage(std::string filename) { try { - std::ifstream tmp(filename, std::ios::binary | std::ios::ate); - dsize_t file_size = tmp.tellg(); - tmp.close(); - - std::ifstream file(filename, std::ios::binary); - TensorShape in_shape({file_size}); - raw_input_tensor_ = std::make_shared(in_shape, DataType(DataType::DE_UINT8)); - - file.read(reinterpret_cast(raw_input_tensor_->GetMutableBuffer()), raw_input_tensor_->SizeInBytes()); + Tensor::CreateTensor(&raw_input_tensor_, filename); raw_cv_image_ = cv::imread(filename, cv::ImreadModes::IMREAD_COLOR); input_tensor_ = std::dynamic_pointer_cast(std::make_shared(raw_cv_image_)); SwapRedAndBlue(input_tensor_, &input_tensor_); diff --git a/tests/ut/cpp/dataset/image_folder_op_test.cc b/tests/ut/cpp/dataset/image_folder_op_test.cc index cd72d8f18..d143a9e06 100644 --- a/tests/ut/cpp/dataset/image_folder_op_test.cc +++ b/tests/ut/cpp/dataset/image_folder_op_test.cc @@ -69,11 +69,9 @@ std::shared_ptr ImageFolder(int64_t num_works, int64_t rows, int6 Status Create1DTensor(std::shared_ptr *sample_ids, int64_t num_elements, unsigned char *data = nullptr, DataType::Type data_type = DataType::DE_UINT32) { TensorShape shape(std::vector(1, num_elements)); - RETURN_IF_NOT_OK( - Tensor::CreateTensor(sample_ids, TensorImpl::kFlexible, shape, DataType(data_type), data)); - if (data == nullptr) { - (*sample_ids)->GetMutableBuffer(); // allocate memory in case user forgets! - } + RETURN_IF_NOT_OK(Tensor::CreateTensor(sample_ids, TensorImpl::kFlexible, shape, DataType(data_type), data)); + (*sample_ids)->AllocateBuffer((*sample_ids)->SizeInBytes()); // allocate memory in case user forgets! + return Status::OK(); } diff --git a/tests/ut/cpp/dataset/map_op_test.cc b/tests/ut/cpp/dataset/map_op_test.cc index 881c71109..21394d400 100644 --- a/tests/ut/cpp/dataset/map_op_test.cc +++ b/tests/ut/cpp/dataset/map_op_test.cc @@ -189,7 +189,7 @@ TEST_F(MindDataTestMapOp, TestByPosition) { EXPECT_EQ(tensor_list[i]->type(), golden_types[i]); EXPECT_EQ(tensor_list[i]->Rank(), golden_ranks[i]); EXPECT_EQ(tensor_list[i]->shape(), golden_shapes[i]); - EXPECT_NE(tensor_list[i]->GetMutableBuffer(), nullptr); + EXPECT_NE(tensor_list[i]->GetBuffer(), nullptr); } } @@ -365,7 +365,7 @@ TEST_F(MindDataTestMapOp, Test1to3) { EXPECT_EQ(tensor_list[i]->type(), golden_types[i]); EXPECT_EQ(tensor_list[i]->Rank(), golden_ranks[i]); EXPECT_EQ(tensor_list[i]->shape(), golden_shapes[i]); - EXPECT_NE(tensor_list[i]->GetMutableBuffer(), nullptr); + EXPECT_NE(tensor_list[i]->GetBuffer(), nullptr); } rc = di.FetchNextTensorRow(&tensor_list); EXPECT_TRUE(rc.IsOk()); @@ -699,7 +699,7 @@ TEST_F(MindDataTestMapOp, ImageFolder_Decode_Repeat_Resize) { MS_LOG(DEBUG) << "row:" << i << "\tlabel:" << label << "\n"; EXPECT_TRUE(img_class[(i % 44) / 11] == label); // Dump all the image into string, to be used as a comparison later. - result.append((char *)tensor_map["image"]->GetMutableBuffer(), (int64_t) tensor_map["image"]->Size()); + result.append((char *)tensor_map["image"]->GetBuffer(), (int64_t)tensor_map["image"]->Size()); di.GetNextAsMap(&tensor_map); i++; } @@ -744,7 +744,7 @@ TEST_F(MindDataTestMapOp, ImageFolder_Decode_Repeat_Resize) { tensor_map["label"]->GetItemAt(&label, {}); MS_LOG(DEBUG) << "row:" << i << "\tlabel:" << label << "\n"; EXPECT_TRUE(img_class[(i % 44) / 11] == label); - result2.append((char *)tensor_map["image"]->GetMutableBuffer(), (int64_t) tensor_map["image"]->Size()); + result2.append((char *)tensor_map["image"]->GetBuffer(), (int64_t)tensor_map["image"]->Size()); di2.GetNextAsMap(&tensor_map); i++; } diff --git a/tests/ut/cpp/dataset/random_crop_decode_resize_op_test.cc b/tests/ut/cpp/dataset/random_crop_decode_resize_op_test.cc index 42db95922..1c9f3a98d 100644 --- a/tests/ut/cpp/dataset/random_crop_decode_resize_op_test.cc +++ b/tests/ut/cpp/dataset/random_crop_decode_resize_op_test.cc @@ -54,16 +54,17 @@ TEST_F(MindDataTestRandomCropDecodeResizeOp, TestOp2) { auto decode_and_crop = static_cast(crop_and_decode_copy); EXPECT_TRUE(crop_and_decode.OneToOne()); GlobalContext::config_manager()->set_seed(42); - for (int i = 0; i < 100; i++) { + for (int k = 0; k < 100; k++) { (void)crop_and_decode.Compute(raw_input_tensor_, &crop_and_decode_output); (void)decode_and_crop.Compute(input_tensor_, &decode_and_crop_output); - cv::Mat output1(target_height, target_width, CV_8UC3, crop_and_decode_output->GetMutableBuffer()); - cv::Mat output2(target_height, target_width, CV_8UC3, decode_and_crop_output->GetMutableBuffer()); + cv::Mat output1 = CVTensor::AsCVTensor(crop_and_decode_output)->mat().clone(); + cv::Mat output2 = CVTensor::AsCVTensor(decode_and_crop_output)->mat().clone(); + long int mse_sum = 0; long int count = 0; int a, b; - for (int j = 0; j < target_height; j++) { - for (int k = 0; k < target_width; k++) { + for (int i = 0; i < target_height; i++) { + for (int j = 0; j < target_width; j++) { a = static_cast(output1.at(i, j)[1]); b = static_cast(output2.at(i, j)[1]); mse_sum += sqrt((a - b) * (a - b)); @@ -133,8 +134,8 @@ TEST_F(MindDataTestRandomCropDecodeResizeOp, TestOp1) { crop_and_decode_status = Crop(decoded, &decoded_and_cropped, x, y, crop_width, crop_height); decode_and_crop_status = JpegCropAndDecode(raw_input_tensor_, &cropped_and_decoded, x, y, crop_width, crop_height); { - cv::Mat M1(crop_height, crop_width, CV_8UC3, decoded_and_cropped->GetMutableBuffer()); - cv::Mat M2(crop_height, crop_width, CV_8UC3, cropped_and_decoded->GetMutableBuffer()); + cv::Mat M1 = CVTensor::AsCVTensor(decoded_and_cropped)->mat().clone(); + cv::Mat M2 = CVTensor::AsCVTensor(cropped_and_decoded)->mat().clone(); for (int i = 0; i < crop_height; ++i) { for (int j = 0; j < crop_width; ++j) { m1 = M1.at(i, j)[1]; diff --git a/tests/ut/cpp/dataset/stand_alone_samplers_test.cc b/tests/ut/cpp/dataset/stand_alone_samplers_test.cc index 0c11ef9dc..03107a43b 100644 --- a/tests/ut/cpp/dataset/stand_alone_samplers_test.cc +++ b/tests/ut/cpp/dataset/stand_alone_samplers_test.cc @@ -31,11 +31,9 @@ using namespace mindspore::dataset; Status CreateINT64Tensor(std::shared_ptr *sample_ids, int64_t num_elements, unsigned char *data = nullptr) { TensorShape shape(std::vector(1, num_elements)); - RETURN_IF_NOT_OK(Tensor::CreateTensor(sample_ids, TensorImpl::kFlexible, shape, - DataType(DataType::DE_INT64), data)); - if (data == nullptr) { - (*sample_ids)->GetMutableBuffer(); // allocate memory in case user forgets! - } + RETURN_IF_NOT_OK(Tensor::CreateTensor(sample_ids, TensorImpl::kFlexible, shape, DataType(DataType::DE_INT64), data)); + (*sample_ids)->AllocateBuffer((*sample_ids)->SizeInBytes()); // allocate memory in case user forgets! + return Status::OK(); } diff --git a/tests/ut/cpp/dataset/tensor_string_test.cc b/tests/ut/cpp/dataset/tensor_string_test.cc index a440a93c1..248080968 100644 --- a/tests/ut/cpp/dataset/tensor_string_test.cc +++ b/tests/ut/cpp/dataset/tensor_string_test.cc @@ -69,10 +69,10 @@ TEST_F(MindDataTestStringTensorDE, Basics2) { std::vector offsets = {0, 4, 9, 12, 18, 22, 26}; uint32_t ctr = 0; for (auto i : offsets) { - ASSERT_TRUE(*(reinterpret_cast(t->GetMutableBuffer() + ctr)) == i + 28); + ASSERT_TRUE(*(reinterpret_cast(t->GetBuffer() + ctr)) == i + 28); ctr += 4; } - const char *buf = reinterpret_cast(t->GetMutableBuffer()) + 6 * 4 + 4; + const char *buf = reinterpret_cast(t->GetBuffer()) + 6 * 4 + 4; std::vector starts = {0, 4, 9, 12, 18, 22}; uint32_t index = 0; @@ -94,10 +94,10 @@ TEST_F(MindDataTestStringTensorDE, Empty) { std::vector offsets = {0, 4, 9, 10, 11, 15, 16}; uint32_t ctr = 0; for (auto i : offsets) { - ASSERT_TRUE(*(reinterpret_cast(t->GetMutableBuffer() + ctr)) == i + 28); + ASSERT_TRUE(*(reinterpret_cast(t->GetBuffer() + ctr)) == i + 28); ctr += 4; } - const char *buf = reinterpret_cast(t->GetMutableBuffer()) + 6 * 4 + 4; + const char *buf = reinterpret_cast(t->GetBuffer()) + 6 * 4 + 4; std::vector starts = {0, 4, 9, 10, 11, 15}; uint32_t index = 0; diff --git a/tests/ut/cpp/dataset/tensor_test.cc b/tests/ut/cpp/dataset/tensor_test.cc index b36f71f4e..6c7402c6b 100644 --- a/tests/ut/cpp/dataset/tensor_test.cc +++ b/tests/ut/cpp/dataset/tensor_test.cc @@ -71,8 +71,9 @@ TEST_F(MindDataTestTensorDE, Basics) { ASSERT_TRUE(rc.IsError()); ASSERT_EQ(t->ToString(), "Tensor (shape: <2,3>, Type: uint64)\n[[1,2,3],[4,5,6]]"); std::vector x = {1, 2, 3, 4, 5, 6}; - std::shared_ptr t2 = std::make_shared(TensorShape({2, 3}), DataType(DataType::DE_UINT64), - reinterpret_cast(&x[0])); + std::shared_ptr t2; + Tensor::CreateTensor(&t2, x, TensorShape({2, 3})); + ASSERT_EQ(*t == *t2, true); ASSERT_EQ(*t != *t2, false); } @@ -81,8 +82,8 @@ TEST_F(MindDataTestTensorDE, Fill) { std::shared_ptr t = std::make_shared(TensorShape({2, 2}), DataType(DataType::DE_FLOAT32)); t->Fill(2.5); std::vector x = {2.5, 2.5, 2.5, 2.5}; - std::shared_ptr t2 = std::make_shared(TensorShape({2, 2}), DataType(DataType::DE_FLOAT32), - reinterpret_cast(&x[0])); + std::shared_ptr t2; + Tensor::CreateTensor(&t2, x, TensorShape({2, 2})); ASSERT_EQ(*t == *t2, true); } @@ -91,8 +92,9 @@ TEST_F(MindDataTestTensorDE, Reshape) { t->Fill(254); t->Reshape(TensorShape({4})); std::vector x = {254, 254, 254, 254}; - std::shared_ptr t2 = std::make_shared(TensorShape({4}), DataType(DataType::DE_UINT8), - reinterpret_cast(&x[0])); + std::shared_ptr t2; + Tensor::CreateTensor(&t2, x); + ASSERT_EQ(*t == *t2, true); Status rc = t->Reshape(TensorShape({5})); ASSERT_TRUE(rc.IsError()); @@ -112,17 +114,17 @@ TEST_F(MindDataTestTensorDE, CopyTensor) { int16_t o; t->GetItemAt(&o, {}); ASSERT_EQ(o, -66); - unsigned char *addr = t->GetMutableBuffer(); + const unsigned char *addr = t->GetBuffer(); auto t2 = std::make_shared(std::move(*t)); ASSERT_EQ(t2->shape(), TensorShape({})); ASSERT_EQ(t2->type(), DataType::DE_INT16); t2->GetItemAt(&o, {}); ASSERT_EQ(o, -66); - unsigned char *new_addr = t2->GetMutableBuffer(); + const unsigned char *new_addr = t2->GetBuffer(); ASSERT_EQ(addr, new_addr); ASSERT_EQ(t->shape(), TensorShape::CreateUnknownRankShape()); ASSERT_EQ(t->type(), DataType::DE_UNKNOWN); - ASSERT_EQ(t->GetMutableBuffer(), nullptr); + ASSERT_EQ(t->GetBuffer(), nullptr); Status rc = t->GetItemAt(&o, {}); ASSERT_TRUE(rc.IsError()); } @@ -130,26 +132,30 @@ TEST_F(MindDataTestTensorDE, CopyTensor) { TEST_F(MindDataTestTensorDE, InsertTensor) { std::shared_ptr t = std::make_shared(TensorShape({2, 3}), DataType(DataType::DE_FLOAT64)); std::vector x = {1.1, 2.1, 3.1}; - std::shared_ptr t2 = std::make_shared(TensorShape({3}), DataType(DataType::DE_FLOAT64), - reinterpret_cast(&x[0])); + std::shared_ptr t2; + Tensor::CreateTensor(&t2, x); + std::vector y = {1.2, 2.2, 3.2}; - std::shared_ptr t3 = std::make_shared(TensorShape({3}), DataType(DataType::DE_FLOAT64), - reinterpret_cast(&y[0])); + std::shared_ptr t3; + Tensor::CreateTensor(&t3, y); + ASSERT_TRUE(t->InsertTensor({0}, t2).OK()); ASSERT_TRUE(t->InsertTensor({1}, t3).OK()); std::vector z = {1.1, 2.1, 3.1, 1.2, 2.2, 3.2}; - std::shared_ptr t4 = std::make_shared(TensorShape({2, 3}), DataType(DataType::DE_FLOAT64), - reinterpret_cast(&z[0])); + std::shared_ptr t4; + Tensor::CreateTensor(&t4, z, TensorShape({2, 3})); + ASSERT_EQ(*t == *t4, true); - std::vector x2 = {0}; - std::shared_ptr t5 = std::make_shared(TensorShape({}), DataType(DataType::DE_FLOAT64), - reinterpret_cast(&x2[0])); + std::shared_ptr t5; + Tensor::CreateTensor(&t5, 0); + ASSERT_TRUE(t->InsertTensor({1, 2}, t5).OK()); z[5] = 0; - std::shared_ptr t6 = std::make_shared(TensorShape({2, 3}), DataType(DataType::DE_FLOAT64), - reinterpret_cast(&z[0])); + std::shared_ptr t6; + Tensor::CreateTensor(&t6, z, TensorShape({2, 3})); + ASSERT_EQ(*t == *t6, true); ASSERT_EQ(t->InsertTensor({2}, t5).get_code(), StatusCode::kUnexpectedError); ASSERT_EQ(t->InsertTensor({1}, t5).get_code(), StatusCode::kUnexpectedError); @@ -238,12 +244,11 @@ TEST_F(MindDataTestTensorDE, Strides) { void checkCvMat(TensorShape shape, DataType type) { std::shared_ptr t = std::make_shared(shape, type); cv::Mat m = t->mat(); - ASSERT_EQ(m.data, t->GetMutableBuffer()); + ASSERT_EQ(m.data, t->GetBuffer()); ASSERT_EQ(static_cast(m.type()) & static_cast(CV_MAT_DEPTH_MASK), type.AsCVType()); if (shape.Rank() < 4) { if (shape.Rank() > 1) { - for (dsize_t i = 0; i < 2; i++) - ASSERT_EQ(m.size[static_cast(i)], shape[i]); + for (dsize_t i = 0; i < 2; i++) ASSERT_EQ(m.size[static_cast(i)], shape[i]); } else if (shape.Rank() == 0) { ASSERT_EQ(m.size[0], 1); ASSERT_EQ(m.size[1], 1); @@ -312,15 +317,15 @@ TEST_F(MindDataTestTensorDE, CVTensorFromMat) { TEST_F(MindDataTestTensorDE, CVTensorAs) { std::shared_ptr t = std::make_shared(TensorShape({3, 2}), DataType(DataType::DE_FLOAT64)); t->Fill(2.2); - unsigned char *addr = t->GetMutableBuffer(); + const unsigned char *addr = t->GetBuffer(); std::shared_ptr t2 = std::make_shared(TensorShape({3, 2}), DataType(DataType::DE_FLOAT64)); t2->Fill(4.4); std::shared_ptr ctv = CVTensor::AsCVTensor(t); - ASSERT_EQ(t->GetMutableBuffer(), nullptr); - ASSERT_EQ(ctv->GetMutableBuffer(), addr); + ASSERT_EQ(t->GetBuffer(), nullptr); + ASSERT_EQ(ctv->GetBuffer(), addr); cv::Mat m = ctv->mat(); m = 2 * m; - ASSERT_EQ(ctv->GetMutableBuffer(), addr); + ASSERT_EQ(ctv->GetBuffer(), addr); ASSERT_TRUE(*t2 == *ctv); MS_LOG(DEBUG) << *t2 << std::endl << *ctv; } @@ -357,8 +362,9 @@ TEST_F(MindDataTestTensorDE, TensorIterator) { std::vector values = {1, 2, 3, 4, 5, 6}; std::vector values2 = {2, 3, 4, 5, 6, 7}; - std::shared_ptr t = std::make_shared(TensorShape({6}), DataType(DataType::DE_UINT32), - reinterpret_cast(&values[0])); + std::shared_ptr t; + Tensor::CreateTensor(&t, values); + auto i = t->begin(); auto j = values.begin(); uint32_t ctr = 0; @@ -367,7 +373,7 @@ TEST_F(MindDataTestTensorDE, TensorIterator) { ctr++; } ASSERT_TRUE(ctr == 6); - t->Reshape(TensorShape {2, 3}); + t->Reshape(TensorShape{2, 3}); i = t->begin(); j = values.begin(); ctr = 0; -- GitLab