From 373adaa5531f137d359627ed4387bc14423097ed Mon Sep 17 00:00:00 2001 From: luoyang Date: Thu, 13 Aug 2020 19:27:08 +0800 Subject: [PATCH] !4402 [Dataset] Fix c_api bugs --- .../ccsrc/minddata/dataset/api/datasets.cc | 14 ++++--- .../ccsrc/minddata/dataset/api/iterator.cc | 4 +- .../ccsrc/minddata/dataset/include/datasets.h | 1 + .../ut/cpp/dataset/c_api_dataset_ops_test.cc | 42 +++++++++++++------ tests/ut/cpp/dataset/c_api_datasets_test.cc | 22 ++++++++++ 5 files changed, 63 insertions(+), 20 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/api/datasets.cc b/mindspore/ccsrc/minddata/dataset/api/datasets.cc index 6021b6336..cc0e18a84 100644 --- a/mindspore/ccsrc/minddata/dataset/api/datasets.cc +++ b/mindspore/ccsrc/minddata/dataset/api/datasets.cc @@ -397,6 +397,10 @@ bool ValidateCommonDatasetParams(std::string dataset_dir) { MS_LOG(ERROR) << "No dataset path is specified"; return false; } + if (access(dataset_dir.c_str(), R_OK) == -1) { + MS_LOG(ERROR) << "No access to specified dataset path: " << dataset_dir; + return false; + } return true; } @@ -1176,8 +1180,8 @@ std::vector> RepeatDataset::Build() { } bool RepeatDataset::ValidateParams() { - if (repeat_count_ != -1 && repeat_count_ <= 0) { - MS_LOG(ERROR) << "Repeat: Repeat count cannot be" << repeat_count_; + if (repeat_count_ <= 0 && repeat_count_ != -1) { + MS_LOG(ERROR) << "Repeat: repeat_count should be either -1 or positive integer, repeat_count_: " << repeat_count_; return false; } @@ -1223,7 +1227,7 @@ std::vector> SkipDataset::Build() { // Function to validate the parameters for SkipDataset bool SkipDataset::ValidateParams() { if (skip_count_ <= -1) { - MS_LOG(ERROR) << "Skip: Invalid input, skip_count: " << skip_count_; + MS_LOG(ERROR) << "Skip: skip_count should not be negative, skip_count: " << skip_count_; return false; } @@ -1244,8 +1248,8 @@ std::vector> TakeDataset::Build() { // Function to validate the parameters for TakeDataset bool TakeDataset::ValidateParams() { - if (take_count_ < -1) { - MS_LOG(ERROR) << "Take: Invalid input, take_count: " << take_count_; + if (take_count_ < 0 && take_count_ != -1) { + MS_LOG(ERROR) << "Take: take_count should be either -1 or positive integer, take_count: " << take_count_; return false; } diff --git a/mindspore/ccsrc/minddata/dataset/api/iterator.cc b/mindspore/ccsrc/minddata/dataset/api/iterator.cc index ea6f463a0..df1b8e7b5 100644 --- a/mindspore/ccsrc/minddata/dataset/api/iterator.cc +++ b/mindspore/ccsrc/minddata/dataset/api/iterator.cc @@ -25,7 +25,7 @@ namespace api { void Iterator::GetNextRow(TensorMap *row) { Status rc = iterator_->GetNextAsMap(row); if (rc.IsError()) { - MS_LOG(ERROR) << "GetNextRow: Failed to get next row."; + MS_LOG(ERROR) << "GetNextRow: Failed to get next row. Error status: " << rc; row->clear(); } } @@ -35,7 +35,7 @@ void Iterator::GetNextRow(TensorVec *row) { TensorRow tensor_row; Status rc = iterator_->FetchNextTensorRow(&tensor_row); if (rc.IsError()) { - MS_LOG(ERROR) << "GetNextRow: Failed to get next row."; + MS_LOG(ERROR) << "GetNextRow: Failed to get next row. Error status: " << rc; row->clear(); } // Generate a vector as return diff --git a/mindspore/ccsrc/minddata/dataset/include/datasets.h b/mindspore/ccsrc/minddata/dataset/include/datasets.h index 1904fa15d..da6456f7e 100644 --- a/mindspore/ccsrc/minddata/dataset/include/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/datasets.h @@ -17,6 +17,7 @@ #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASETS_H_ #define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASETS_H_ +#include #include #include #include diff --git a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc index 1c900598d..1a45d0a2f 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc @@ -437,22 +437,22 @@ TEST_F(MindDataTestPipeline, TestRepeatDefault) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_NE(ds,nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds // Default value of repeat count is -1, expected to repeat infinitely ds = ds->Repeat(); - EXPECT_NE(ds,nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 1; ds = ds->Batch(batch_size); - EXPECT_NE(ds,nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_NE(iter,nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map > row; @@ -460,14 +460,16 @@ TEST_F(MindDataTestPipeline, TestRepeatDefault) { uint64_t i = 0; while (row.size()!= 0) { // manually stop - if(i==100){break;} + if (i == 100) { + break; + } i++; auto image = row["image"]; MS_LOG(INFO)<< "Tensor image shape: " << image->shape(); iter->GetNextRow(&row); } - EXPECT_EQ(i,100); + EXPECT_EQ(i, 100); // Manually terminate the pipeline iter->Stop(); } @@ -478,22 +480,22 @@ TEST_F(MindDataTestPipeline, TestRepeatOne) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_NE(ds,nullptr); + EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds int32_t repeat_num = 1; ds = ds->Repeat(repeat_num); - EXPECT_NE(ds,nullptr); + EXPECT_NE(ds, nullptr); // Create a Batch operation on ds int32_t batch_size = 1; ds = ds->Batch(batch_size); - EXPECT_NE(ds,nullptr); + EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_NE(iter,nullptr); + EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::unordered_map > row; @@ -506,13 +508,27 @@ TEST_F(MindDataTestPipeline, TestRepeatOne) { iter->GetNextRow(&row); } - EXPECT_EQ(i,10); + EXPECT_EQ(i, 10); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestRepeatFail) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRepeatFail."; +TEST_F(MindDataTestPipeline, TestRepeatFail1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRepeatFail1."; + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + + // Create a Repeat operation on ds + int32_t repeat_num = 0; + ds = ds->Repeat(repeat_num); + EXPECT_EQ(ds, nullptr); +} + +TEST_F(MindDataTestPipeline, TestRepeatFail2) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRepeatFail2."; // This case is expected to fail because the repeat count is invalid (<-1 && !=0). // Create an ImageFolder Dataset diff --git a/tests/ut/cpp/dataset/c_api_datasets_test.cc b/tests/ut/cpp/dataset/c_api_datasets_test.cc index 31acdc770..1ae562a61 100644 --- a/tests/ut/cpp/dataset/c_api_datasets_test.cc +++ b/tests/ut/cpp/dataset/c_api_datasets_test.cc @@ -133,3 +133,25 @@ TEST_F(MindDataTestPipeline, TestMnistFail1) { std::shared_ptr ds = Mnist("", RandomSampler(false, 10)); EXPECT_EQ(ds, nullptr); } + +TEST_F(MindDataTestPipeline, TestImageFolderFail2) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestImageFolderFail2."; + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 2), {".JGP"}); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + EXPECT_EQ(row.size(), 0); + + // Manually terminate the pipeline + iter->Stop(); +} -- GitLab