From deb114647487e8d94d45c3040aec72a18b3ddbc7 Mon Sep 17 00:00:00 2001 From: tony_liu2 Date: Mon, 10 Aug 2020 14:27:40 -0400 Subject: [PATCH] reorganize headers and tests remove headers fix iterator file fix pr comments fix new file --- .../cpp/dataset/c_api_dataset_cifar_test.cc | 42 +- .../ut/cpp/dataset/c_api_dataset_clue_test.cc | 348 +++++----- .../ut/cpp/dataset/c_api_dataset_coco_test.cc | 152 ++--- .../dataset/c_api_dataset_filetext_test.cc | 284 ++++---- .../dataset/c_api_dataset_iterator_test.cc | 156 ++--- .../ut/cpp/dataset/c_api_dataset_ops_test.cc | 646 +++++++++--------- .../ut/cpp/dataset/c_api_dataset_voc_test.cc | 154 ++--- tests/ut/cpp/dataset/c_api_datasets_test.cc | 57 +- tests/ut/cpp/dataset/c_api_samplers_test.cc | 22 - tests/ut/cpp/dataset/c_api_transforms_test.cc | 314 ++++----- 10 files changed, 979 insertions(+), 1196 deletions(-) diff --git a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc index 7f248c5f1..2e4125516 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc @@ -13,37 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include -#include - -#include "utils/log_adapter.h" -#include "utils/ms_utils.h" #include "common/common.h" -#include "gtest/gtest.h" -#include "securec.h" #include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/include/status.h" -#include "minddata/dataset/include/transforms.h" -#include "minddata/dataset/include/iterator.h" -#include "minddata/dataset/core/constants.h" -#include "minddata/dataset/core/tensor_shape.h" -#include "minddata/dataset/core/tensor.h" -#include "minddata/dataset/include/samplers.h" using namespace mindspore::dataset::api; -using mindspore::MsLogLevel::ERROR; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::LogStream; using mindspore::dataset::Tensor; -using mindspore::dataset::TensorShape; -using mindspore::dataset::TensorImpl; -using mindspore::dataset::DataType; -using mindspore::dataset::Status; -using mindspore::dataset::BorderType; -using mindspore::dataset::dsize_t; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: @@ -83,14 +57,6 @@ TEST_F(MindDataTestPipeline, TestCifar10Dataset) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestCifar10DatasetFail1) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10DatasetFail1."; - - // Create a Cifar10 Dataset - std::shared_ptr ds = Cifar10("", RandomSampler(false, 10)); - EXPECT_EQ(ds, nullptr); -} - TEST_F(MindDataTestPipeline, TestCifar100Dataset) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar100Dataset."; @@ -133,3 +99,11 @@ TEST_F(MindDataTestPipeline, TestCifar100DatasetFail1) { std::shared_ptr ds = Cifar100("", RandomSampler(false, 10)); EXPECT_EQ(ds, nullptr); } + +TEST_F(MindDataTestPipeline, TestCifar10DatasetFail1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10DatasetFail1."; + + // Create a Cifar10 Dataset + std::shared_ptr ds = Cifar10("", RandomSampler(false, 10)); + EXPECT_EQ(ds, nullptr); +} diff --git a/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc b/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc index d57fae3fe..4c6ca885c 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc @@ -14,9 +14,9 @@ * limitations under the License. */ #include "common/common.h" -#include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/core/global_context.h" #include "minddata/dataset/core/config_manager.h" +#include "minddata/dataset/core/global_context.h" +#include "minddata/dataset/include/datasets.h" using namespace mindspore::dataset::api; using mindspore::dataset::ShuffleMode; @@ -27,76 +27,6 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; -TEST_F(MindDataTestPipeline, TestCLUEDatasetBasic) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetBasic."; - - // Create a CLUEFile Dataset, with single CLUE file - std::string clue_file = datasets_root_path_ + "/testCLUE/afqmc/train.json"; - std::string task = "AFQMC"; - std::string usage = "train"; - std::shared_ptr ds = CLUE({clue_file}, task, usage, 2); - EXPECT_NE(ds, nullptr); - - // Create an iterator over the result of the above dataset - // This will trigger the creation of the Execution Tree and launch it. - std::shared_ptr iter = ds->CreateIterator(); - EXPECT_NE(iter, nullptr); - - // Iterate the dataset and get each row - std::unordered_map> row; - iter->GetNextRow(&row); - - EXPECT_NE(row.find("sentence1"), row.end()); - uint64_t i = 0; - while (row.size() != 0) { - auto text = row["sentence1"]; - MS_LOG(INFO) << "Tensor text shape: " << text->shape(); - i++; - iter->GetNextRow(&row); - } - - // Expect 2 samples - EXPECT_EQ(i, 2); - - // Manually terminate the pipeline - iter->Stop(); -} - -TEST_F(MindDataTestPipeline, TestCLUEDatasetDistribution) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetDistribution."; - - // Create a CLUEFile Dataset, with single CLUE file - std::string clue_file = datasets_root_path_ + "/testCLUE/afqmc/train.json"; - std::string task = "AFQMC"; - std::string usage = "train"; - std::shared_ptr ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, 3, 0); - EXPECT_NE(ds, nullptr); - - // Create an iterator over the result of the above dataset - // This will trigger the creation of the Execution Tree and launch it. - std::shared_ptr iter = ds->CreateIterator(); - EXPECT_NE(iter, nullptr); - - // Iterate the dataset and get each row - std::unordered_map> row; - iter->GetNextRow(&row); - - EXPECT_NE(row.find("sentence1"), row.end()); - uint64_t i = 0; - while (row.size() != 0) { - auto text = row["sentence1"]; - MS_LOG(INFO) << "Tensor text shape: " << text->shape(); - i++; - iter->GetNextRow(&row); - } - - // Expect 1 samples - EXPECT_EQ(i, 1); - - // Manually terminate the pipeline - iter->Stop(); -} - TEST_F(MindDataTestPipeline, TestCLUEDatasetAFQMC) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetAFQMC."; @@ -194,6 +124,41 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetAFQMC) { iter->Stop(); } +TEST_F(MindDataTestPipeline, TestCLUEDatasetBasic) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetBasic."; + + // Create a CLUEFile Dataset, with single CLUE file + std::string clue_file = datasets_root_path_ + "/testCLUE/afqmc/train.json"; + std::string task = "AFQMC"; + std::string usage = "train"; + std::shared_ptr ds = CLUE({clue_file}, task, usage, 2); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); + + EXPECT_NE(row.find("sentence1"), row.end()); + uint64_t i = 0; + while (row.size() != 0) { + auto text = row["sentence1"]; + MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + i++; + iter->GetNextRow(&row); + } + + // Expect 2 samples + EXPECT_EQ(i, 2); + + // Manually terminate the pipeline + iter->Stop(); +} + TEST_F(MindDataTestPipeline, TestCLUEDatasetCMNLI) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetCMNLI."; @@ -284,14 +249,14 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetCSL) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestCLUEDatasetIFLYTEK) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetIFLYTEK."; +TEST_F(MindDataTestPipeline, TestCLUEDatasetDistribution) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetDistribution."; // Create a CLUEFile Dataset, with single CLUE file - std::string clue_file = datasets_root_path_ + "/testCLUE/iflytek/train.json"; - std::string task = "IFLYTEK"; + std::string clue_file = datasets_root_path_ + "/testCLUE/afqmc/train.json"; + std::string task = "AFQMC"; std::string usage = "train"; - std::shared_ptr ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kFalse); + std::shared_ptr ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, 3, 0); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -303,38 +268,61 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetIFLYTEK) { std::unordered_map> row; iter->GetNextRow(&row); - EXPECT_NE(row.find("sentence"), row.end()); - std::vector expected_result = { - "第一个文本", - "第二个文本", - "第三个文本" - }; - + EXPECT_NE(row.find("sentence1"), row.end()); uint64_t i = 0; while (row.size() != 0) { - auto text = row["sentence"]; - std::string_view sv; - text->GetItemAt(&sv, {0}); - std::string ss(sv); - EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); + auto text = row["sentence1"]; MS_LOG(INFO) << "Tensor text shape: " << text->shape(); - iter->GetNextRow(&row); i++; + iter->GetNextRow(&row); } - // Expect 3 samples - EXPECT_EQ(i, 3); + // Expect 1 samples + EXPECT_EQ(i, 1); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestCLUEDatasetTNEWS) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetTNEWS."; +TEST_F(MindDataTestPipeline, TestCLUEDatasetException) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetException."; + // Create a CLUE Dataset + std::string clue_file = datasets_root_path_ + "/testCLUE/wsc/train.json"; + std::string task = "WSC"; + std::string usage = "train"; + std::string invalid_clue_file = "./NotExistFile"; + + std::shared_ptr ds0 = CLUE({}, task, usage); + EXPECT_EQ(ds0, nullptr); + + std::shared_ptr ds1 = CLUE({invalid_clue_file}, task, usage); + EXPECT_EQ(ds1, nullptr); + + std::shared_ptr ds2 = CLUE({clue_file}, "invalid_task", usage); + EXPECT_EQ(ds2, nullptr); + + std::shared_ptr ds3 = CLUE({clue_file}, task, "invalid_usage"); + EXPECT_EQ(ds3, nullptr); + + std::shared_ptr ds4 = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, 2, 2); + EXPECT_EQ(ds4, nullptr); + + std::shared_ptr ds5 = CLUE({clue_file}, task, usage, -1, ShuffleMode::kGlobal); + EXPECT_EQ(ds5, nullptr); + + std::shared_ptr ds6 = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, -1); + EXPECT_EQ(ds6, nullptr); + + std::shared_ptr ds7 = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, 0, -1); + EXPECT_EQ(ds7, nullptr); +} + +TEST_F(MindDataTestPipeline, TestCLUEDatasetIFLYTEK) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetIFLYTEK."; // Create a CLUEFile Dataset, with single CLUE file - std::string clue_file = datasets_root_path_ + "/testCLUE/tnews/train.json"; - std::string task = "TNEWS"; + std::string clue_file = datasets_root_path_ + "/testCLUE/iflytek/train.json"; + std::string task = "IFLYTEK"; std::string usage = "train"; std::shared_ptr ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kFalse); EXPECT_NE(ds, nullptr); @@ -350,9 +338,9 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetTNEWS) { EXPECT_NE(row.find("sentence"), row.end()); std::vector expected_result = { - "新闻1", - "新闻2", - "新闻3" + "第一个文本", + "第二个文本", + "第三个文本" }; uint64_t i = 0; @@ -374,17 +362,31 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetTNEWS) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestCLUEDatasetWSC) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetWSC."; +TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleFiles) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetShuffleFiles."; + // Test CLUE Dataset with files shuffle, num_parallel_workers=1 - // Create a CLUEFile Dataset, with single CLUE file - std::string clue_file = datasets_root_path_ + "/testCLUE/wsc/train.json"; - std::string task = "WSC"; + // Set configuration + uint32_t original_seed = GlobalContext::config_manager()->seed(); + uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers(); + MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers; + GlobalContext::config_manager()->set_seed(135); + GlobalContext::config_manager()->set_num_parallel_workers(1); + + // Create a CLUE Dataset, with two text files + // Note: train.json has 3 rows + // Note: dev.json has 3 rows + // Use default of all samples + // They have the same keywords + // Set shuffle to files shuffle + std::string clue_file1 = datasets_root_path_ + "/testCLUE/afqmc/train.json"; + std::string clue_file2 = datasets_root_path_ + "/testCLUE/afqmc/dev.json"; + std::string task = "AFQMC"; std::string usage = "train"; - std::shared_ptr ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kFalse); + std::shared_ptr ds = CLUE({clue_file1, clue_file2}, task, usage, 0, ShuffleMode::kFiles); EXPECT_NE(ds, nullptr); - // Create an iterator over the result of the above dataset + // Create an iterator over the result of the above dataset. // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); EXPECT_NE(iter, nullptr); @@ -393,30 +395,37 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetWSC) { std::unordered_map> row; iter->GetNextRow(&row); - EXPECT_NE(row.find("text"), row.end()); + EXPECT_NE(row.find("sentence1"), row.end()); std::vector expected_result = { - "小明呢,他在哪?", - "小红刚刚看到小明,他在操场", - "等小明回来,小张你叫他交作业" + "蚂蚁借呗等额还款能否换成先息后本", + "蚂蚁花呗说我违约了", + "帮我看看本月花呗账单结清了没", + "你有花呗吗", + "吃饭能用花呗吗", + "蚂蚁花呗支付金额有什么限制" }; uint64_t i = 0; while (row.size() != 0) { - auto text = row["text"]; + auto text = row["sentence1"]; std::string_view sv; text->GetItemAt(&sv, {0}); std::string ss(sv); + // Compare against expected result EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); - MS_LOG(INFO) << "Tensor text shape: " << text->shape(); - iter->GetNextRow(&row); i++; + iter->GetNextRow(&row); } - // Expect 3 samples - EXPECT_EQ(i, 3); + // Expect 3 + 3 = 6 samples + EXPECT_EQ(i, 6); // Manually terminate the pipeline iter->Stop(); + + // Restore configuration + GlobalContext::config_manager()->set_seed(original_seed); + GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers); } TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleGlobal) { @@ -476,31 +485,17 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleGlobal) { GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers); } -TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleFiles) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetShuffleFiles."; - // Test CLUE Dataset with files shuffle, num_parallel_workers=1 - - // Set configuration - uint32_t original_seed = GlobalContext::config_manager()->seed(); - uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers(); - MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers; - GlobalContext::config_manager()->set_seed(135); - GlobalContext::config_manager()->set_num_parallel_workers(1); +TEST_F(MindDataTestPipeline, TestCLUEDatasetTNEWS) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetTNEWS."; - // Create a CLUE Dataset, with two text files - // Note: train.json has 3 rows - // Note: dev.json has 3 rows - // Use default of all samples - // They have the same keywords - // Set shuffle to files shuffle - std::string clue_file1 = datasets_root_path_ + "/testCLUE/afqmc/train.json"; - std::string clue_file2 = datasets_root_path_ + "/testCLUE/afqmc/dev.json"; - std::string task = "AFQMC"; + // Create a CLUEFile Dataset, with single CLUE file + std::string clue_file = datasets_root_path_ + "/testCLUE/tnews/train.json"; + std::string task = "TNEWS"; std::string usage = "train"; - std::shared_ptr ds = CLUE({clue_file1, clue_file2}, task, usage, 0, ShuffleMode::kFiles); + std::shared_ptr ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kFalse); EXPECT_NE(ds, nullptr); - // Create an iterator over the result of the above dataset. + // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); EXPECT_NE(iter, nullptr); @@ -509,68 +504,73 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleFiles) { std::unordered_map> row; iter->GetNextRow(&row); - EXPECT_NE(row.find("sentence1"), row.end()); + EXPECT_NE(row.find("sentence"), row.end()); std::vector expected_result = { - "蚂蚁借呗等额还款能否换成先息后本", - "蚂蚁花呗说我违约了", - "帮我看看本月花呗账单结清了没", - "你有花呗吗", - "吃饭能用花呗吗", - "蚂蚁花呗支付金额有什么限制" + "新闻1", + "新闻2", + "新闻3" }; uint64_t i = 0; while (row.size() != 0) { - auto text = row["sentence1"]; + auto text = row["sentence"]; std::string_view sv; text->GetItemAt(&sv, {0}); std::string ss(sv); - // Compare against expected result EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); - i++; + MS_LOG(INFO) << "Tensor text shape: " << text->shape(); iter->GetNextRow(&row); + i++; } - // Expect 3 + 3 = 6 samples - EXPECT_EQ(i, 6); + // Expect 3 samples + EXPECT_EQ(i, 3); // Manually terminate the pipeline iter->Stop(); - - // Restore configuration - GlobalContext::config_manager()->set_seed(original_seed); - GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers); } -TEST_F(MindDataTestPipeline, TestCLUEDatasetException) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetException."; - // Create a CLUE Dataset +TEST_F(MindDataTestPipeline, TestCLUEDatasetWSC) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetWSC."; + + // Create a CLUEFile Dataset, with single CLUE file std::string clue_file = datasets_root_path_ + "/testCLUE/wsc/train.json"; std::string task = "WSC"; std::string usage = "train"; - std::string invalid_clue_file = "./NotExistFile"; - - std::shared_ptr ds0 = CLUE({}, task, usage); - EXPECT_EQ(ds0, nullptr); - - std::shared_ptr ds1 = CLUE({invalid_clue_file}, task, usage); - EXPECT_EQ(ds1, nullptr); + std::shared_ptr ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); - std::shared_ptr ds2 = CLUE({clue_file}, "invalid_task", usage); - EXPECT_EQ(ds2, nullptr); + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); - std::shared_ptr ds3 = CLUE({clue_file}, task, "invalid_usage"); - EXPECT_EQ(ds3, nullptr); + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); - std::shared_ptr ds4 = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, 2, 2); - EXPECT_EQ(ds4, nullptr); + EXPECT_NE(row.find("text"), row.end()); + std::vector expected_result = { + "小明呢,他在哪?", + "小红刚刚看到小明,他在操场", + "等小明回来,小张你叫他交作业" + }; - std::shared_ptr ds5 = CLUE({clue_file}, task, usage, -1, ShuffleMode::kGlobal); - EXPECT_EQ(ds5, nullptr); + uint64_t i = 0; + while (row.size() != 0) { + auto text = row["text"]; + std::string_view sv; + text->GetItemAt(&sv, {0}); + std::string ss(sv); + EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); + MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + iter->GetNextRow(&row); + i++; + } - std::shared_ptr ds6 = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, -1); - EXPECT_EQ(ds6, nullptr); + // Expect 3 samples + EXPECT_EQ(i, 3); - std::shared_ptr ds7 = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, 0, -1); - EXPECT_EQ(ds7, nullptr); + // Manually terminate the pipeline + iter->Stop(); } diff --git a/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc b/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc index b4986c53c..dcc21e253 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc @@ -13,49 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include -#include - -#include "utils/log_adapter.h" -#include "utils/ms_utils.h" #include "common/common.h" -#include "gtest/gtest.h" -#include "securec.h" #include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/include/status.h" -#include "minddata/dataset/include/transforms.h" -#include "minddata/dataset/include/iterator.h" -#include "minddata/dataset/core/constants.h" -#include "minddata/dataset/core/tensor_shape.h" -#include "minddata/dataset/core/tensor.h" -#include "minddata/dataset/include/samplers.h" using namespace mindspore::dataset::api; -using mindspore::MsLogLevel::ERROR; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::LogStream; using mindspore::dataset::Tensor; using mindspore::dataset::TensorShape; -using mindspore::dataset::TensorImpl; -using mindspore::dataset::DataType; -using mindspore::dataset::Status; -using mindspore::dataset::BorderType; using mindspore::dataset::dsize_t; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; -TEST_F(MindDataTestPipeline, TestCocoDetection) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoDetection."; +TEST_F(MindDataTestPipeline, TestCocoDefault) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoDefault."; // Create a Coco Dataset std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; - std::shared_ptr ds = Coco(folder_path, annotation_file, "Detection", false, SequentialSampler(0, 6)); + std::shared_ptr ds = Coco(folder_path, annotation_file); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -67,28 +43,14 @@ TEST_F(MindDataTestPipeline, TestCocoDetection) { std::unordered_map> row; iter->GetNextRow(&row); - std::string expect_file[] = {"000000391895", "000000318219", "000000554625", "000000574769", "000000060623", - "000000309022"}; - std::vector> expect_bbox_vector = {{10.0, 10.0, 10.0, 10.0, 70.0, 70.0, 70.0, 70.0}, - {20.0, 20.0, 20.0, 20.0, 80.0, 80.0, 80.0, 80.0}, - {30.0, 30.0, 30.0, 30.0}, {40.0, 40.0, 40.0, 40.0}, - {50.0, 50.0, 50.0, 50.0}, {60.0, 60.0, 60.0, 60.0}}; - std::vector> expect_catagoryid_list = {{1, 7}, {2, 8}, {3}, {4}, {5}, {6}}; uint64_t i = 0; while (row.size() != 0) { auto image = row["image"]; auto bbox = row["bbox"]; auto category_id = row["category_id"]; - std::shared_ptr expect_image; - Tensor::CreateFromFile(folder_path + "/" + expect_file[i] + ".jpg", &expect_image); - EXPECT_EQ(*image, *expect_image); - std::shared_ptr expect_bbox; - dsize_t bbox_num = static_cast(expect_bbox_vector[i].size() / 4); - Tensor::CreateFromVector(expect_bbox_vector[i], TensorShape({bbox_num, 4}), &expect_bbox); - EXPECT_EQ(*bbox, *expect_bbox); - std::shared_ptr expect_categoryid; - Tensor::CreateFromVector(expect_catagoryid_list[i], TensorShape({bbox_num, 1}), &expect_categoryid); - EXPECT_EQ(*category_id, *expect_categoryid); + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + MS_LOG(INFO) << "Tensor bbox shape: " << bbox->shape(); + MS_LOG(INFO) << "Tensor category_id shape: " << category_id->shape(); iter->GetNextRow(&row); i++; } @@ -99,13 +61,13 @@ TEST_F(MindDataTestPipeline, TestCocoDetection) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestCocoStuff) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoStuff."; +TEST_F(MindDataTestPipeline, TestCocoDetection) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoDetection."; // Create a Coco Dataset std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; - std::shared_ptr ds = Coco(folder_path, annotation_file, "Stuff", false, SequentialSampler(0, 6)); + std::shared_ptr ds = Coco(folder_path, annotation_file, "Detection", false, SequentialSampler(0, 6)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -119,27 +81,26 @@ TEST_F(MindDataTestPipeline, TestCocoStuff) { std::string expect_file[] = {"000000391895", "000000318219", "000000554625", "000000574769", "000000060623", "000000309022"}; - std::vector> expect_segmentation_vector = - {{10.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, - 70.0, 72.0, 73.0, 74.0, 75.0, -1.0, -1.0, -1.0, -1.0, -1.0}, - {20.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, - 10.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, -1.0}, - {40.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 40.0, 41.0, 42.0}, - {50.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, 58.0, 59.0, 60.0, 61.0, 62.0, 63.0}, - {60.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0}, - {60.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0}}; - std::vector> expect_size = {{2, 10}, {2, 11}, {1, 12}, {1, 13}, {1, 14}, {2, 7}}; + std::vector> expect_bbox_vector = {{10.0, 10.0, 10.0, 10.0, 70.0, 70.0, 70.0, 70.0}, + {20.0, 20.0, 20.0, 20.0, 80.0, 80.0, 80.0, 80.0}, + {30.0, 30.0, 30.0, 30.0}, {40.0, 40.0, 40.0, 40.0}, + {50.0, 50.0, 50.0, 50.0}, {60.0, 60.0, 60.0, 60.0}}; + std::vector> expect_catagoryid_list = {{1, 7}, {2, 8}, {3}, {4}, {5}, {6}}; uint64_t i = 0; while (row.size() != 0) { auto image = row["image"]; - auto segmentation = row["segmentation"]; - auto iscrowd = row["iscrowd"]; + auto bbox = row["bbox"]; + auto category_id = row["category_id"]; std::shared_ptr expect_image; Tensor::CreateFromFile(folder_path + "/" + expect_file[i] + ".jpg", &expect_image); EXPECT_EQ(*image, *expect_image); - std::shared_ptr expect_segmentation; - Tensor::CreateFromVector(expect_segmentation_vector[i], TensorShape(expect_size[i]), &expect_segmentation); - EXPECT_EQ(*segmentation, *expect_segmentation); + std::shared_ptr expect_bbox; + dsize_t bbox_num = static_cast(expect_bbox_vector[i].size() / 4); + Tensor::CreateFromVector(expect_bbox_vector[i], TensorShape({bbox_num, 4}), &expect_bbox); + EXPECT_EQ(*bbox, *expect_bbox); + std::shared_ptr expect_categoryid; + Tensor::CreateFromVector(expect_catagoryid_list[i], TensorShape({bbox_num, 1}), &expect_categoryid); + EXPECT_EQ(*category_id, *expect_categoryid); iter->GetNextRow(&row); i++; } @@ -150,6 +111,24 @@ TEST_F(MindDataTestPipeline, TestCocoStuff) { iter->Stop(); } +TEST_F(MindDataTestPipeline, TestCocoException) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoException."; + // Create a Coco Dataset + std::string folder_path = datasets_root_path_ + "/testCOCO/train"; + std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; + std::string invalid_folder_path = "./NotExist"; + std::string invalid_annotation_file = "./NotExistFile"; + + std::shared_ptr ds = Coco(invalid_folder_path, annotation_file); + EXPECT_EQ(ds, nullptr); + + std::shared_ptr ds1 = Coco(folder_path, invalid_annotation_file); + EXPECT_EQ(ds1, nullptr); + + std::shared_ptr ds2 = Coco(folder_path, annotation_file, "valid_mode"); + EXPECT_EQ(ds2, nullptr); +} + TEST_F(MindDataTestPipeline, TestCocoKeypoint) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoKeypoint."; // Create a Coco Dataset @@ -261,13 +240,13 @@ TEST_F(MindDataTestPipeline, TestCocoPanoptic) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestCocoDefault) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoDetection."; +TEST_F(MindDataTestPipeline, TestCocoStuff) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoStuff."; // Create a Coco Dataset std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; - std::shared_ptr ds = Coco(folder_path, annotation_file); + std::shared_ptr ds = Coco(folder_path, annotation_file, "Stuff", false, SequentialSampler(0, 6)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -279,14 +258,29 @@ TEST_F(MindDataTestPipeline, TestCocoDefault) { std::unordered_map> row; iter->GetNextRow(&row); + std::string expect_file[] = {"000000391895", "000000318219", "000000554625", "000000574769", "000000060623", + "000000309022"}; + std::vector> expect_segmentation_vector = + {{10.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, + 70.0, 72.0, 73.0, 74.0, 75.0, -1.0, -1.0, -1.0, -1.0, -1.0}, + {20.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, + 10.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, -1.0}, + {40.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 40.0, 41.0, 42.0}, + {50.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, 58.0, 59.0, 60.0, 61.0, 62.0, 63.0}, + {60.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0}, + {60.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0}}; + std::vector> expect_size = {{2, 10}, {2, 11}, {1, 12}, {1, 13}, {1, 14}, {2, 7}}; uint64_t i = 0; while (row.size() != 0) { auto image = row["image"]; - auto bbox = row["bbox"]; - auto category_id = row["category_id"]; - MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - MS_LOG(INFO) << "Tensor bbox shape: " << bbox->shape(); - MS_LOG(INFO) << "Tensor category_id shape: " << category_id->shape(); + auto segmentation = row["segmentation"]; + auto iscrowd = row["iscrowd"]; + std::shared_ptr expect_image; + Tensor::CreateFromFile(folder_path + "/" + expect_file[i] + ".jpg", &expect_image); + EXPECT_EQ(*image, *expect_image); + std::shared_ptr expect_segmentation; + Tensor::CreateFromVector(expect_segmentation_vector[i], TensorShape(expect_size[i]), &expect_segmentation); + EXPECT_EQ(*segmentation, *expect_segmentation); iter->GetNextRow(&row); i++; } @@ -296,21 +290,3 @@ TEST_F(MindDataTestPipeline, TestCocoDefault) { // Manually terminate the pipeline iter->Stop(); } - -TEST_F(MindDataTestPipeline, TestCocoException) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoDetection."; - // Create a Coco Dataset - std::string folder_path = datasets_root_path_ + "/testCOCO/train"; - std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; - std::string invalid_folder_path = "./NotExist"; - std::string invalid_annotation_file = "./NotExistFile"; - - std::shared_ptr ds = Coco(invalid_folder_path, annotation_file); - EXPECT_EQ(ds, nullptr); - - std::shared_ptr ds1 = Coco(folder_path, invalid_annotation_file); - EXPECT_EQ(ds1, nullptr); - - std::shared_ptr ds2 = Coco(folder_path, annotation_file, "valid_mode"); - EXPECT_EQ(ds2, nullptr); -} diff --git a/tests/ut/cpp/dataset/c_api_dataset_filetext_test.cc b/tests/ut/cpp/dataset/c_api_dataset_filetext_test.cc index 5ce5794e2..6e27a14e2 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_filetext_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_filetext_test.cc @@ -13,41 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include -#include - -#include "utils/log_adapter.h" -#include "utils/ms_utils.h" #include "common/common.h" -#include "gtest/gtest.h" -#include "./securec.h" - -#include "minddata/dataset/core/client.h" #include "minddata/dataset/core/config_manager.h" -#include "minddata/dataset/core/constants.h" #include "minddata/dataset/core/global_context.h" -#include "minddata/dataset/core/tensor.h" -#include "minddata/dataset/core/tensor_shape.h" #include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/include/iterator.h" -#include "minddata/dataset/include/samplers.h" -#include "minddata/dataset/include/status.h" -#include "minddata/dataset/include/transforms.h" using namespace mindspore::dataset; using namespace mindspore::dataset::api; -using mindspore::LogStream; -using mindspore::dataset::DataType; using mindspore::dataset::ShuffleMode; -using mindspore::dataset::Status; using mindspore::dataset::Tensor; -using mindspore::dataset::TensorImpl; -using mindspore::dataset::TensorShape; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::MsLogLevel::ERROR; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: @@ -109,6 +83,87 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetBasic) { GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers); } +TEST_F(MindDataTestPipeline, TestTextFileDatasetFail1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail1."; + + // Attempt to create a TextFile Dataset + // with invalid samplers=-1 + std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; + std::shared_ptr ds = TextFile({tf_file1}, -1); + + // Expect failure: Number of samples cannot be negative + EXPECT_EQ(ds, nullptr); +} + +TEST_F(MindDataTestPipeline, TestTextFileDatasetFail2) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail2."; + + // Attempt to create a TextFile Dataset + // with wrongful empty dataset_files input + std::shared_ptr ds = TextFile({}); + + // Expect failure: dataset_files is not specified + EXPECT_EQ(ds, nullptr); +} + +TEST_F(MindDataTestPipeline, TestTextFileDatasetFail3) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail3."; + + // Attempt to create a TextFile Dataset + // with non-existent dataset_files input + std::shared_ptr ds = TextFile({"notexist.txt"}, 0, ShuffleMode::kFalse); + + // Expect failure: specified dataset_files does not exist + EXPECT_EQ(ds, nullptr); +} + +TEST_F(MindDataTestPipeline, TestTextFileDatasetFail4) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail4."; + + // Attempt to create a TextFile Dataset + // with empty string dataset_files input + std::shared_ptr ds = TextFile({""}, 0, ShuffleMode::kFiles); + + // Expect failure: specified dataset_files does not exist + EXPECT_EQ(ds, nullptr); +} + +TEST_F(MindDataTestPipeline, TestTextFileDatasetFail5) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail5."; + + // Attempt to create a TextFile Dataset + // with invalid num_shards=0 value + std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; + std::shared_ptr ds = TextFile({tf_file1}, 1, ShuffleMode::kFalse, 0); + + // Expect failure: Number of shards cannot be <=0 + EXPECT_EQ(ds, nullptr); +} + +TEST_F(MindDataTestPipeline, TestTextFileDatasetFail6) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail6."; + + // Attempt to create a TextFile Dataset + // with invalid shard_id=-1 value + std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; + std::shared_ptr ds = TextFile({tf_file1}, 0, ShuffleMode::kFiles, -1); + + // Expect failure: shard_id cannot be negative + EXPECT_EQ(ds, nullptr); +} + +TEST_F(MindDataTestPipeline, TestTextFileDatasetFail7) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail7."; + + // Attempt to create a TextFile Dataset + // with invalid shard_id=2 and num_shards=2 combination + std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; + std::shared_ptr ds = TextFile({tf_file1}, 0, ShuffleMode::kGlobal, 2, 2); + + // Expect failure: Cannot have shard_id >= num_shards + EXPECT_EQ(ds, nullptr); +} + TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse1) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleFalse1."; // Test TextFile Dataset with two text files and no shuffle, num_parallel_workers=1 @@ -224,22 +279,25 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse4Shard) { GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers); } -TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleGlobal1A."; - // Test TextFile Dataset with 1 text file, global shuffle, num_parallel_workers=1 +TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleFiles1."; + // Test TextFile Dataset with files shuffle, num_parallel_workers=1 // Set configuration uint32_t original_seed = GlobalContext::config_manager()->seed(); uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers(); MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers; - GlobalContext::config_manager()->set_seed(246); + GlobalContext::config_manager()->set_seed(135); GlobalContext::config_manager()->set_num_parallel_workers(1); // Create a TextFile Dataset, with two text files // Note: 1.txt has 3 rows - // Set shuffle to global shuffle + // Note: 2.txt has 2 rows + // Use default of all samples + // Set shuffle to files shuffle std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; - std::shared_ptr ds = TextFile({tf_file1}, 0, ShuffleMode::kGlobal); + std::string tf_file2 = datasets_root_path_ + "/testTextFileDataset/2.txt"; + std::shared_ptr ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kFiles); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset. @@ -252,7 +310,9 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) { iter->GetNextRow(&row); EXPECT_NE(row.find("text"), row.end()); - std::vector expected_result = {"Good luck to everyone.", "This is a text file.", "Be happy every day."}; + std::vector expected_result = { + "This is a text file.", "Be happy every day.", "Good luck to everyone.", "Another file.", "End of file.", + }; uint64_t i = 0; while (row.size() != 0) { @@ -268,8 +328,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) { iter->GetNextRow(&row); } - // Expect 3 samples - EXPECT_EQ(i, 3); + // Expect 2 + 3 = 5 samples + EXPECT_EQ(i, 5); // Manually terminate the pipeline iter->Stop(); @@ -279,24 +339,25 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) { GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers); } -TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleGlobal1B."; - // Test TextFile Dataset with 2 text files, global shuffle, num_parallel_workers=1 +TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleFiles4."; + // Test TextFile Dataset with files shuffle, num_parallel_workers=4 // Set configuration uint32_t original_seed = GlobalContext::config_manager()->seed(); uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers(); MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers; - GlobalContext::config_manager()->set_seed(246); - GlobalContext::config_manager()->set_num_parallel_workers(1); + GlobalContext::config_manager()->set_seed(135); + GlobalContext::config_manager()->set_num_parallel_workers(4); // Create a TextFile Dataset, with two text files // Note: 1.txt has 3 rows // Note: 2.txt has 2 rows - // Set shuffle to global shuffle + // Use default of all samples + // Set shuffle to files shuffle std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; std::string tf_file2 = datasets_root_path_ + "/testTextFileDataset/2.txt"; - std::shared_ptr ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kGlobal); + std::shared_ptr ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kFiles); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset. @@ -309,8 +370,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) { iter->GetNextRow(&row); EXPECT_NE(row.find("text"), row.end()); - std::vector expected_result = {"Another file.", "Good luck to everyone.", "This is a text file.", - "End of file.", "Be happy every day."}; + std::vector expected_result = {"This is a text file.", "Another file.", "Be happy every day.", + "End of file.", "Good luck to everyone."}; uint64_t i = 0; while (row.size() != 0) { @@ -337,24 +398,22 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) { GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers); } -TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleGlobal4."; - // Test TextFile Dataset with 2 text files, global shuffle, num_parallel_workers=4 +TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleGlobal1A."; + // Test TextFile Dataset with 1 text file, global shuffle, num_parallel_workers=1 // Set configuration uint32_t original_seed = GlobalContext::config_manager()->seed(); uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers(); MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers; GlobalContext::config_manager()->set_seed(246); - GlobalContext::config_manager()->set_num_parallel_workers(4); + GlobalContext::config_manager()->set_num_parallel_workers(1); // Create a TextFile Dataset, with two text files // Note: 1.txt has 3 rows - // Note: 2.txt has 2 rows // Set shuffle to global shuffle std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; - std::string tf_file2 = datasets_root_path_ + "/testTextFileDataset/2.txt"; - std::shared_ptr ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kGlobal); + std::shared_ptr ds = TextFile({tf_file1}, 0, ShuffleMode::kGlobal); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset. @@ -367,8 +426,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) { iter->GetNextRow(&row); EXPECT_NE(row.find("text"), row.end()); - std::vector expected_result = {"Another file.", "Good luck to everyone.", "End of file.", - "This is a text file.", "Be happy every day."}; + std::vector expected_result = {"Good luck to everyone.", "This is a text file.", "Be happy every day."}; uint64_t i = 0; while (row.size() != 0) { @@ -384,8 +442,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) { iter->GetNextRow(&row); } - // Expect 2 + 3 = 5 samples - EXPECT_EQ(i, 5); + // Expect 3 samples + EXPECT_EQ(i, 3); // Manually terminate the pipeline iter->Stop(); @@ -395,25 +453,24 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) { GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers); } -TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleFiles1."; - // Test TextFile Dataset with files shuffle, num_parallel_workers=1 +TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleGlobal1B."; + // Test TextFile Dataset with 2 text files, global shuffle, num_parallel_workers=1 // Set configuration uint32_t original_seed = GlobalContext::config_manager()->seed(); uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers(); MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers; - GlobalContext::config_manager()->set_seed(135); + GlobalContext::config_manager()->set_seed(246); GlobalContext::config_manager()->set_num_parallel_workers(1); // Create a TextFile Dataset, with two text files // Note: 1.txt has 3 rows // Note: 2.txt has 2 rows - // Use default of all samples - // Set shuffle to files shuffle + // Set shuffle to global shuffle std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; std::string tf_file2 = datasets_root_path_ + "/testTextFileDataset/2.txt"; - std::shared_ptr ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kFiles); + std::shared_ptr ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kGlobal); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset. @@ -426,9 +483,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1) { iter->GetNextRow(&row); EXPECT_NE(row.find("text"), row.end()); - std::vector expected_result = { - "This is a text file.", "Be happy every day.", "Good luck to everyone.", "Another file.", "End of file.", - }; + std::vector expected_result = {"Another file.", "Good luck to everyone.", "This is a text file.", + "End of file.", "Be happy every day."}; uint64_t i = 0; while (row.size() != 0) { @@ -455,25 +511,24 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1) { GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers); } -TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleFiles4."; - // Test TextFile Dataset with files shuffle, num_parallel_workers=4 +TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleGlobal4."; + // Test TextFile Dataset with 2 text files, global shuffle, num_parallel_workers=4 // Set configuration uint32_t original_seed = GlobalContext::config_manager()->seed(); uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers(); MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers; - GlobalContext::config_manager()->set_seed(135); + GlobalContext::config_manager()->set_seed(246); GlobalContext::config_manager()->set_num_parallel_workers(4); // Create a TextFile Dataset, with two text files // Note: 1.txt has 3 rows // Note: 2.txt has 2 rows - // Use default of all samples - // Set shuffle to files shuffle + // Set shuffle to global shuffle std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; std::string tf_file2 = datasets_root_path_ + "/testTextFileDataset/2.txt"; - std::shared_ptr ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kFiles); + std::shared_ptr ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kGlobal); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset. @@ -486,8 +541,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) { iter->GetNextRow(&row); EXPECT_NE(row.find("text"), row.end()); - std::vector expected_result = {"This is a text file.", "Another file.", "Be happy every day.", - "End of file.", "Good luck to everyone."}; + std::vector expected_result = {"Another file.", "Good luck to everyone.", "End of file.", + "This is a text file.", "Be happy every day."}; uint64_t i = 0; while (row.size() != 0) { @@ -513,84 +568,3 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) { GlobalContext::config_manager()->set_seed(original_seed); GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers); } - -TEST_F(MindDataTestPipeline, TestTextFileDatasetFail1) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail1."; - - // Attempt to create a TextFile Dataset - // with invalid samplers=-1 - std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; - std::shared_ptr ds = TextFile({tf_file1}, -1); - - // Expect failure: Number of samples cannot be negative - EXPECT_EQ(ds, nullptr); -} - -TEST_F(MindDataTestPipeline, TestTextFileDatasetFail2) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail2."; - - // Attempt to create a TextFile Dataset - // with wrongful empty dataset_files input - std::shared_ptr ds = TextFile({}); - - // Expect failure: dataset_files is not specified - EXPECT_EQ(ds, nullptr); -} - -TEST_F(MindDataTestPipeline, TestTextFileDatasetFail3) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail3."; - - // Attempt to create a TextFile Dataset - // with non-existent dataset_files input - std::shared_ptr ds = TextFile({"notexist.txt"}, 0, ShuffleMode::kFalse); - - // Expect failure: specified dataset_files does not exist - EXPECT_EQ(ds, nullptr); -} - -TEST_F(MindDataTestPipeline, TestTextFileDatasetFail4) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail4."; - - // Attempt to create a TextFile Dataset - // with empty string dataset_files input - std::shared_ptr ds = TextFile({""}, 0, ShuffleMode::kFiles); - - // Expect failure: specified dataset_files does not exist - EXPECT_EQ(ds, nullptr); -} - -TEST_F(MindDataTestPipeline, TestTextFileDatasetFail5) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail5."; - - // Attempt to create a TextFile Dataset - // with invalid num_shards=0 value - std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; - std::shared_ptr ds = TextFile({tf_file1}, 1, ShuffleMode::kFalse, 0); - - // Expect failure: Number of shards cannot be <=0 - EXPECT_EQ(ds, nullptr); -} - -TEST_F(MindDataTestPipeline, TestTextFileDatasetFail6) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail6."; - - // Attempt to create a TextFile Dataset - // with invalid shard_id=-1 value - std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; - std::shared_ptr ds = TextFile({tf_file1}, 0, ShuffleMode::kFiles, -1); - - // Expect failure: shard_id cannot be negative - EXPECT_EQ(ds, nullptr); -} - -TEST_F(MindDataTestPipeline, TestTextFileDatasetFail7) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail7."; - - // Attempt to create a TextFile Dataset - // with invalid shard_id=2 and num_shards=2 combination - std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt"; - std::shared_ptr ds = TextFile({tf_file1}, 0, ShuffleMode::kGlobal, 2, 2); - - // Expect failure: Cannot have shard_id >= num_shards - EXPECT_EQ(ds, nullptr); -} diff --git a/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc b/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc index a6fbe7a62..9612b8311 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc @@ -13,139 +13,107 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include -#include - -#include "utils/log_adapter.h" -#include "utils/ms_utils.h" #include "common/common.h" -#include "gtest/gtest.h" -#include "securec.h" #include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/include/status.h" -#include "minddata/dataset/include/transforms.h" -#include "minddata/dataset/include/iterator.h" -#include "minddata/dataset/core/constants.h" -#include "minddata/dataset/core/tensor_shape.h" -#include "minddata/dataset/core/tensor.h" -#include "minddata/dataset/include/samplers.h" using namespace mindspore::dataset::api; -using mindspore::MsLogLevel::ERROR; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::LogStream; using mindspore::dataset::Tensor; using mindspore::dataset::TensorShape; -using mindspore::dataset::TensorImpl; -using mindspore::dataset::DataType; -using mindspore::dataset::Status; -using mindspore::dataset::BorderType; -using mindspore::dataset::dsize_t; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; - -TEST_F(MindDataTestPipeline, TestIteratorOneColumn) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn."; - // Create a Mnist Dataset - std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, RandomSampler(false, 4)); +TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorEmptyColumn."; + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, RandomSampler(false, 5)); EXPECT_NE(ds, nullptr); - // Create a Batch operation on ds - int32_t batch_size = 2; - ds = ds->Batch(batch_size); + // Create a Rename operation on ds + ds = ds->Rename({"image", "label"}, {"col1", "col2"}); EXPECT_NE(ds, nullptr); - // Create an iterator over the result of the above dataset - // Only select "image" column and drop others - std::vector columns = {"image"}; - std::shared_ptr iter = ds->CreateIterator(columns); + // No columns are specified, use all columns + std::shared_ptr iter = ds->CreateIterator(); EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::vector> row; iter->GetNextRow(&row); - TensorShape expect({2, 28, 28, 1}); + TensorShape expect0({32, 32, 3}); + TensorShape expect1({}); uint64_t i = 0; while (row.size() != 0) { - for (auto &v : row) { - MS_LOG(INFO) << "image shape:" << v->shape(); - EXPECT_EQ(expect, v->shape()); - } + MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape(); + EXPECT_EQ(expect0, row[0]->shape()); + EXPECT_EQ(expect1, row[1]->shape()); iter->GetNextRow(&row); i++; } - EXPECT_EQ(i, 2); + EXPECT_EQ(i, 5); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorTwoColumns."; - // Create a VOC Dataset - std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, false, SequentialSampler(0, 4)); +TEST_F(MindDataTestPipeline, TestIteratorOneColumn) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn."; + // Create a Mnist Dataset + std::string folder_path = datasets_root_path_ + "/testMnistData/"; + std::shared_ptr ds = Mnist(folder_path, RandomSampler(false, 4)); EXPECT_NE(ds, nullptr); - // Create a Repeat operation on ds - int32_t repeat_num = 2; - ds = ds->Repeat(repeat_num); + // Create a Batch operation on ds + int32_t batch_size = 2; + ds = ds->Batch(batch_size); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset - // Only select "image" and "bbox" column - std::vector columns = {"image", "bbox"}; + // Only select "image" column and drop others + std::vector columns = {"image"}; std::shared_ptr iter = ds->CreateIterator(columns); EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::vector> row; iter->GetNextRow(&row); - std::vector expect = {TensorShape({173673}), TensorShape({1, 4}), - TensorShape({173673}), TensorShape({1, 4}), - TensorShape({147025}), TensorShape({1, 4}), - TensorShape({211653}), TensorShape({1, 4})}; + TensorShape expect({2, 28, 28, 1}); uint64_t i = 0; - uint64_t j = 0; while (row.size() != 0) { - MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape(); - EXPECT_EQ(2, row.size()); - EXPECT_EQ(expect[j++], row[0]->shape()); - EXPECT_EQ(expect[j++], row[1]->shape()); + for (auto &v : row) { + MS_LOG(INFO) << "image shape:" << v->shape(); + EXPECT_EQ(expect, v->shape()); + } iter->GetNextRow(&row); i++; - j = (j == expect.size()) ? 0 : j; } - EXPECT_EQ(i, 8); + EXPECT_EQ(i, 2); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorEmptyColumn."; +TEST_F(MindDataTestPipeline, TestIteratorReOrder) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorReOrder."; // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, RandomSampler(false, 5)); + std::shared_ptr ds = Cifar10(folder_path, SequentialSampler(false, 4)); EXPECT_NE(ds, nullptr); - // Create a Rename operation on ds - ds = ds->Rename({"image", "label"}, {"col1", "col2"}); + // Create a Take operation on ds + ds = ds->Take(2); EXPECT_NE(ds, nullptr); - // No columns are specified, use all columns - std::shared_ptr iter = ds->CreateIterator(); + // Create an iterator over the result of the above dataset + // Reorder "image" and "label" column + std::vector columns = {"label", "image"}; + std::shared_ptr iter = ds->CreateIterator(columns); EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row @@ -154,56 +122,62 @@ TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) { TensorShape expect0({32, 32, 3}); TensorShape expect1({}); + // Check if we will catch "label" before "image" in row + std::vector expect = {"label", "image"}; uint64_t i = 0; while (row.size() != 0) { MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape(); - EXPECT_EQ(expect0, row[0]->shape()); - EXPECT_EQ(expect1, row[1]->shape()); + EXPECT_EQ(expect1, row[0]->shape()); + EXPECT_EQ(expect0, row[1]->shape()); iter->GetNextRow(&row); i++; } - EXPECT_EQ(i, 5); + EXPECT_EQ(i, 2); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestIteratorReOrder) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorReOrder."; - // Create a Cifar10 Dataset - std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, SequentialSampler(false, 4)); +TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorTwoColumns."; + // Create a VOC Dataset + std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, false, SequentialSampler(0, 4)); EXPECT_NE(ds, nullptr); - // Create a Take operation on ds - ds = ds->Take(2); + // Create a Repeat operation on ds + int32_t repeat_num = 2; + ds = ds->Repeat(repeat_num); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset - // Reorder "image" and "label" column - std::vector columns = {"label", "image"}; + // Only select "image" and "bbox" column + std::vector columns = {"image", "bbox"}; std::shared_ptr iter = ds->CreateIterator(columns); EXPECT_NE(iter, nullptr); // Iterate the dataset and get each row std::vector> row; iter->GetNextRow(&row); - TensorShape expect0({32, 32, 3}); - TensorShape expect1({}); + std::vector expect = {TensorShape({173673}), TensorShape({1, 4}), + TensorShape({173673}), TensorShape({1, 4}), + TensorShape({147025}), TensorShape({1, 4}), + TensorShape({211653}), TensorShape({1, 4})}; - // Check if we will catch "label" before "image" in row - std::vector expect = {"label", "image"}; uint64_t i = 0; + uint64_t j = 0; while (row.size() != 0) { MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape(); - EXPECT_EQ(expect1, row[0]->shape()); - EXPECT_EQ(expect0, row[1]->shape()); + EXPECT_EQ(2, row.size()); + EXPECT_EQ(expect[j++], row[0]->shape()); + EXPECT_EQ(expect[j++], row[1]->shape()); iter->GetNextRow(&row); i++; + j = (j == expect.size()) ? 0 : j; } - EXPECT_EQ(i, 2); + EXPECT_EQ(i, 8); // Manually terminate the pipeline iter->Stop(); diff --git a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc index 23751ccce..ce78ba2b6 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc @@ -13,37 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include -#include - -#include "utils/log_adapter.h" -#include "utils/ms_utils.h" #include "common/common.h" -#include "gtest/gtest.h" -#include "securec.h" #include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/include/status.h" #include "minddata/dataset/include/transforms.h" -#include "minddata/dataset/include/iterator.h" -#include "minddata/dataset/core/constants.h" -#include "minddata/dataset/core/tensor_shape.h" -#include "minddata/dataset/core/tensor.h" -#include "minddata/dataset/include/samplers.h" using namespace mindspore::dataset::api; -using mindspore::MsLogLevel::ERROR; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::LogStream; using mindspore::dataset::Tensor; -using mindspore::dataset::TensorShape; -using mindspore::dataset::TensorImpl; -using mindspore::dataset::DataType; -using mindspore::dataset::Status; -using mindspore::dataset::BorderType; -using mindspore::dataset::dsize_t; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: @@ -90,28 +65,26 @@ TEST_F(MindDataTestPipeline, TestBatchAndRepeat) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTensorOpsAndMap."; - // Create a Mnist Dataset - std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, RandomSampler(false, 20)); - EXPECT_NE(ds, nullptr); +TEST_F(MindDataTestPipeline, TestConcatFail1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatFail1."; + // This case is expected to fail because the input column names of concatenated datasets are not the same - // Create a Repeat operation on ds - int32_t repeat_num = 2; - ds = ds->Repeat(repeat_num); + // Create an ImageFolder Dataset + // Column names: {"image", "label"} + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + std::shared_ptr ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - // Create objects for the tensor ops - std::shared_ptr resize_op = vision::Resize({30, 30}); - EXPECT_NE(resize_op, nullptr); - - std::shared_ptr center_crop_op = vision::CenterCrop({16, 16}); - EXPECT_NE(center_crop_op, nullptr); + // Create a Rename operation on ds + ds2 = ds2->Rename({"image", "label"}, {"col1", "col2"}); + EXPECT_NE(ds, nullptr); - // Create a Map operation on ds - ds = ds->Map({resize_op, center_crop_op}); + // Create a Project operation on the ds + // Name of datasets to concat doesn't not match + ds = ds->Concat({ds2}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -122,41 +95,51 @@ TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) { // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_NE(iter, nullptr); - - // Iterate the dataset and get each row - std::unordered_map> row; - iter->GetNextRow(&row); + EXPECT_EQ(iter, nullptr); +} - uint64_t i = 0; - while (row.size() != 0) { - i++; - auto image = row["image"]; - MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - iter->GetNextRow(&row); - } +TEST_F(MindDataTestPipeline, TestConcatFail2) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatFail2."; + // This case is expected to fail because the input dataset is empty. - EXPECT_EQ(i, 40); + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); - // Manually terminate the pipeline - iter->Stop(); + // Create a Project operation on the ds + // Input dataset to concat is empty + ds = ds->Concat({}); + EXPECT_EQ(ds, nullptr); } -TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestImageFolderBatchAndRepeat."; +TEST_F(MindDataTestPipeline, TestConcatSuccess) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatSuccess."; // Create an ImageFolder Dataset + // Column names: {"image", "label"} std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - // Create a Repeat operation on ds - int32_t repeat_num = 2; - ds = ds->Repeat(repeat_num); + // Create a Cifar10 Dataset + // Column names: {"image", "label"} + folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds2 = Cifar10(folder_path, RandomSampler(false, 9)); + EXPECT_NE(ds2, nullptr); + + // Create a Project operation on ds + ds = ds->Project({"image"}); + EXPECT_NE(ds, nullptr); + ds2 = ds2->Project({"image"}); + EXPECT_NE(ds, nullptr); + + // Create a Concat operation on the ds + ds = ds->Concat({ds2}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds - int32_t batch_size = 2; + int32_t batch_size = 1; ds = ds->Batch(batch_size); EXPECT_NE(ds, nullptr); @@ -168,7 +151,6 @@ TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) { // Iterate the dataset and get each row std::unordered_map> row; iter->GetNextRow(&row); - uint64_t i = 0; while (row.size() != 0) { i++; @@ -177,32 +159,39 @@ TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) { iter->GetNextRow(&row); } - EXPECT_EQ(i, 10); - + EXPECT_EQ(i, 19); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestShuffleDataset) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestShuffleDataset."; +TEST_F(MindDataTestPipeline, TestConcatSuccess2) { + // Test "+" operator to concat two datasets + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatSuccess2."; // Create an ImageFolder Dataset + // Column names: {"image", "label"} std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - // Create a Shuffle operation on ds - int32_t shuffle_size = 10; - ds = ds->Shuffle(shuffle_size); + // Create a Cifar10 Dataset + // Column names: {"image", "label"} + folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds2 = Cifar10(folder_path, RandomSampler(false, 9)); + EXPECT_NE(ds2, nullptr); + + // Create a Project operation on ds + ds = ds->Project({"image"}); + EXPECT_NE(ds, nullptr); + ds2 = ds2->Project({"image"}); EXPECT_NE(ds, nullptr); - // Create a Repeat operation on ds - int32_t repeat_num = 2; - ds = ds->Repeat(repeat_num); + // Create a Concat operation on the ds + ds = ds + ds2; EXPECT_NE(ds, nullptr); // Create a Batch operation on ds - int32_t batch_size = 2; + int32_t batch_size = 1; ds = ds->Batch(batch_size); EXPECT_NE(ds, nullptr); @@ -214,7 +203,6 @@ TEST_F(MindDataTestPipeline, TestShuffleDataset) { // Iterate the dataset and get each row std::unordered_map> row; iter->GetNextRow(&row); - uint64_t i = 0; while (row.size() != 0) { i++; @@ -223,23 +211,27 @@ TEST_F(MindDataTestPipeline, TestShuffleDataset) { iter->GetNextRow(&row); } - EXPECT_EQ(i, 10); - + EXPECT_EQ(i, 19); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestSkipDataset) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSkipDataset."; +TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestImageFolderBatchAndRepeat."; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - // Create a Skip operation on ds - int32_t count = 3; - ds = ds->Skip(count); + // Create a Repeat operation on ds + int32_t repeat_num = 2; + ds = ds->Repeat(repeat_num); + EXPECT_NE(ds, nullptr); + + // Create a Batch operation on ds + int32_t batch_size = 2; + ds = ds->Batch(batch_size); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -258,40 +250,42 @@ TEST_F(MindDataTestPipeline, TestSkipDataset) { MS_LOG(INFO) << "Tensor image shape: " << image->shape(); iter->GetNextRow(&row); } - MS_LOG(INFO) << "Number of rows: " << i; - // Expect 10-3=7 rows - EXPECT_EQ(i, 7); + EXPECT_EQ(i, 10); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestSkipDatasetError1) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSkipDatasetError1."; +TEST_F(MindDataTestPipeline, TestProjectMap) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestProjectMap."; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - // Create a Skip operation on ds with invalid count input - int32_t count = -1; - ds = ds->Skip(count); - // Expect nullptr for invalid input skip_count - EXPECT_EQ(ds, nullptr); -} + // Create a Repeat operation on ds + int32_t repeat_num = 2; + ds = ds->Repeat(repeat_num); + EXPECT_NE(ds, nullptr); -TEST_F(MindDataTestPipeline, TestTakeDatasetDefault) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTakeDatasetDefault."; + // Create objects for the tensor ops + std::shared_ptr random_vertical_flip_op = vision::RandomVerticalFlip(0.5); + EXPECT_NE(random_vertical_flip_op, nullptr); - // Create an ImageFolder Dataset - std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 7)); + // Create a Map operation on ds + ds = ds->Map({random_vertical_flip_op}, {}, {}, {"image", "label"}); EXPECT_NE(ds, nullptr); - // Create a Take operation on ds, dafault count = -1 - ds = ds->Take(); + // Create a Project operation on ds + std::vector column_project = {"image"}; + ds = ds->Project(column_project); + EXPECT_NE(ds, nullptr); + + // Create a Batch operation on ds + int32_t batch_size = 1; + ds = ds->Batch(batch_size); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -310,25 +304,33 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetDefault) { MS_LOG(INFO) << "Tensor image shape: " << image->shape(); iter->GetNextRow(&row); } - MS_LOG(INFO) << "Number of rows: " << i; - // Expect 7 rows - EXPECT_EQ(i, 7); + EXPECT_EQ(i, 20); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestTakeDatasetNormal) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTakeDatasetNormal."; +TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) { + MS_LOG(INFO) << "Doing MindDataTestPipeline.TestProjectMapAutoInjection"; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 8)); + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - // Create a Take operation on ds - ds = ds->Take(5); + // Create a Repeat operation on ds + int32_t repeat_num = 2; + ds = ds->Repeat(repeat_num); + EXPECT_NE(ds, nullptr); + + // Create objects for the tensor ops + std::shared_ptr resize_op = vision::Resize({30, 30}); + EXPECT_NE(resize_op, nullptr); + + // Create a Map operation on ds + // {"image"} is the project columns. This will trigger auto injection of ProjectOp after MapOp. + ds = ds->Map({resize_op}, {}, {}, {"image"}); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -340,39 +342,47 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetNormal) { std::unordered_map> row; iter->GetNextRow(&row); + // 'label' is dropped during the project op + EXPECT_EQ(row.find("label"), row.end()); + // 'image' column should still exist + EXPECT_NE(row.find("image"), row.end()); + uint64_t i = 0; while (row.size() != 0) { i++; auto image = row["image"]; MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + EXPECT_EQ(image->shape()[0], 30); iter->GetNextRow(&row); } - MS_LOG(INFO) << "Number of rows: " << i; - // Expect 5 rows - EXPECT_EQ(i, 5); + EXPECT_EQ(i, 20); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestTakeDatasetError1) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTakeDatasetError1."; +TEST_F(MindDataTestPipeline, TestRenameFail) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRenameFail."; + // We expect this test to fail because input and output in Rename are not the same size // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - // Create a Take operation on ds with invalid count input - int32_t count = -5; - ds = ds->Take(count); - // Expect nullptr for invalid input take_count + // Create a Repeat operation on ds + int32_t repeat_num = 2; + ds = ds->Repeat(repeat_num); + EXPECT_NE(ds, nullptr); + + // Create a Rename operation on ds + ds = ds->Rename({"image", "label"}, {"col2"}); EXPECT_EQ(ds, nullptr); } -TEST_F(MindDataTestPipeline, TestProjectMap) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestProjectMap."; +TEST_F(MindDataTestPipeline, TestRenameSuccess) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRenameSuccess."; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; @@ -384,17 +394,8 @@ TEST_F(MindDataTestPipeline, TestProjectMap) { ds = ds->Repeat(repeat_num); EXPECT_NE(ds, nullptr); - // Create objects for the tensor ops - std::shared_ptr random_vertical_flip_op = vision::RandomVerticalFlip(0.5); - EXPECT_NE(random_vertical_flip_op, nullptr); - - // Create a Map operation on ds - ds = ds->Map({random_vertical_flip_op}, {}, {}, {"image", "label"}); - EXPECT_NE(ds, nullptr); - - // Create a Project operation on ds - std::vector column_project = {"image"}; - ds = ds->Project(column_project); + // Create a Rename operation on ds + ds = ds->Rename({"image", "label"}, {"col1", "col2"}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -412,9 +413,14 @@ TEST_F(MindDataTestPipeline, TestProjectMap) { iter->GetNextRow(&row); uint64_t i = 0; + EXPECT_NE(row.find("col1"), row.end()); + EXPECT_NE(row.find("col2"), row.end()); + EXPECT_EQ(row.find("image"), row.end()); + EXPECT_EQ(row.find("label"), row.end()); + while (row.size() != 0) { i++; - auto image = row["image"]; + auto image = row["col1"]; MS_LOG(INFO) << "Tensor image shape: " << image->shape(); iter->GetNextRow(&row); } @@ -425,27 +431,27 @@ TEST_F(MindDataTestPipeline, TestProjectMap) { iter->Stop(); } - -TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) { - MS_LOG(INFO) << "Doing MindDataTestPipeline.TestProjectMapAutoInjection"; +TEST_F(MindDataTestPipeline, TestShuffleDataset) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestShuffleDataset."; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); + // Create a Shuffle operation on ds + int32_t shuffle_size = 10; + ds = ds->Shuffle(shuffle_size); + EXPECT_NE(ds, nullptr); + // Create a Repeat operation on ds int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); EXPECT_NE(ds, nullptr); - // Create objects for the tensor ops - std::shared_ptr resize_op = vision::Resize({30, 30}); - EXPECT_NE(resize_op, nullptr); - - // Create a Map operation on ds - // {"image"} is the project columns. This will trigger auto injection of ProjectOp after MapOp. - ds = ds->Map({resize_op}, {}, {}, {"image"}); + // Create a Batch operation on ds + int32_t batch_size = 2; + ds = ds->Batch(batch_size); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -457,64 +463,31 @@ TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) { std::unordered_map> row; iter->GetNextRow(&row); - // 'label' is dropped during the project op - EXPECT_EQ(row.find("label"), row.end()); - // 'image' column should still exist - EXPECT_NE(row.find("image"), row.end()); - uint64_t i = 0; while (row.size() != 0) { i++; auto image = row["image"]; MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - EXPECT_EQ(image->shape()[0], 30); iter->GetNextRow(&row); } - EXPECT_EQ(i, 20); + EXPECT_EQ(i, 10); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestZipSuccess) { - // Testing the member zip() function - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipSuccess."; +TEST_F(MindDataTestPipeline, TestSkipDataset) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSkipDataset."; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - // Create a Project operation on ds - std::vector column_project = {"image"}; - ds = ds->Project(column_project); - EXPECT_NE(ds, nullptr); - - // Create an ImageFolder Dataset - std::shared_ptr ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_NE(ds1, nullptr); - - // Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names) - ds1 = ds1->Rename({"image", "label"}, {"col1", "col2"}); - EXPECT_NE(ds1, nullptr); - - folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds2 = Cifar10(folder_path, RandomSampler(false, 10)); - EXPECT_NE(ds2, nullptr); - - // Create a Project operation on ds - column_project = {"label"}; - ds2 = ds2->Project(column_project); - EXPECT_NE(ds2, nullptr); - - // Create a Zip operation on the datasets - ds = ds->Zip({ds1, ds2}); - EXPECT_NE(ds, nullptr); - - // Create a Batch operation on ds - int32_t batch_size = 1; - ds = ds->Batch(batch_size); + // Create a Skip operation on ds + int32_t count = 3; + ds = ds->Skip(count); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -526,13 +499,6 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) { std::unordered_map> row; iter->GetNextRow(&row); - // Check zipped column names - EXPECT_EQ(row.size(), 4); - EXPECT_NE(row.find("image"), row.end()); - EXPECT_NE(row.find("label"), row.end()); - EXPECT_NE(row.find("col1"), row.end()); - EXPECT_NE(row.find("col2"), row.end()); - uint64_t i = 0; while (row.size() != 0) { i++; @@ -540,35 +506,40 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) { MS_LOG(INFO) << "Tensor image shape: " << image->shape(); iter->GetNextRow(&row); } + MS_LOG(INFO) << "Number of rows: " << i; - EXPECT_EQ(i, 10); + // Expect 10-3=7 rows + EXPECT_EQ(i, 7); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestZipSuccess2) { - // Testing the static zip() function - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipSuccess2."; +TEST_F(MindDataTestPipeline, TestSkipDatasetError1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSkipDatasetError1."; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 9)); + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - std::shared_ptr ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_NE(ds2, nullptr); - // Create a Rename operation on ds (so that the 2 datasets we are going to zip have distinct column names) - ds = ds->Rename({"image", "label"}, {"col1", "col2"}); - EXPECT_NE(ds, nullptr); + // Create a Skip operation on ds with invalid count input + int32_t count = -1; + ds = ds->Skip(count); + // Expect nullptr for invalid input skip_count + EXPECT_EQ(ds, nullptr); +} - // Create a Zip operation on the datasets - ds = Zip({ds, ds2}); +TEST_F(MindDataTestPipeline, TestTakeDatasetDefault) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTakeDatasetDefault."; + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 7)); EXPECT_NE(ds, nullptr); - // Create a Batch operation on ds - int32_t batch_size = 1; - ds = ds->Batch(batch_size); + // Create a Take operation on ds, dafault count = -1 + ds = ds->Take(); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -580,13 +551,6 @@ TEST_F(MindDataTestPipeline, TestZipSuccess2) { std::unordered_map> row; iter->GetNextRow(&row); - // Check zipped column names - EXPECT_EQ(row.size(), 4); - EXPECT_NE(row.find("image"), row.end()); - EXPECT_NE(row.find("label"), row.end()); - EXPECT_NE(row.find("col1"), row.end()); - EXPECT_NE(row.find("col2"), row.end()); - uint64_t i = 0; while (row.size() != 0) { i++; @@ -594,63 +558,73 @@ TEST_F(MindDataTestPipeline, TestZipSuccess2) { MS_LOG(INFO) << "Tensor image shape: " << image->shape(); iter->GetNextRow(&row); } + MS_LOG(INFO) << "Number of rows: " << i; - EXPECT_EQ(i, 9); + // Expect 7 rows + EXPECT_EQ(i, 7); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestZipFail) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipFail."; - // We expect this test to fail because we are the both datasets we are zipping have "image" and "label" columns - // and zip doesn't accept datasets with same column names +TEST_F(MindDataTestPipeline, TestTakeDatasetError1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTakeDatasetError1."; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - // Create an ImageFolder Dataset - std::shared_ptr ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_NE(ds1, nullptr); + // Create a Take operation on ds with invalid count input + int32_t count = -5; + ds = ds->Take(count); + // Expect nullptr for invalid input take_count + EXPECT_EQ(ds, nullptr); +} - // Create a Zip operation on the datasets - ds = Zip({ds, ds1}); +TEST_F(MindDataTestPipeline, TestTakeDatasetNormal) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTakeDatasetNormal."; + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 8)); EXPECT_NE(ds, nullptr); - // Create a Batch operation on ds - int32_t batch_size = 1; - ds = ds->Batch(batch_size); + // Create a Take operation on ds + ds = ds->Take(5); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset // This will trigger the creation of the Execution Tree and launch it. std::shared_ptr iter = ds->CreateIterator(); - EXPECT_EQ(iter, nullptr); -} + EXPECT_NE(iter, nullptr); -TEST_F(MindDataTestPipeline, TestZipFail2) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipFail2."; - // This case is expected to fail because the input dataset is empty. + // Iterate the dataset and get each row + std::unordered_map> row; + iter->GetNextRow(&row); - // Create an ImageFolder Dataset - std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_NE(ds, nullptr); + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + iter->GetNextRow(&row); + } + MS_LOG(INFO) << "Number of rows: " << i; - // Create a Zip operation on the datasets - // Input dataset to zip is empty - ds = Zip({}); - EXPECT_EQ(ds, nullptr); + // Expect 5 rows + EXPECT_EQ(i, 5); + + // Manually terminate the pipeline + iter->Stop(); } -TEST_F(MindDataTestPipeline, TestRenameSuccess) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRenameSuccess."; +TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTensorOpsAndMap."; - // Create an ImageFolder Dataset - std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + // Create a Mnist Dataset + std::string folder_path = datasets_root_path_ + "/testMnistData/"; + std::shared_ptr ds = Mnist(folder_path, RandomSampler(false, 20)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -658,8 +632,15 @@ TEST_F(MindDataTestPipeline, TestRenameSuccess) { ds = ds->Repeat(repeat_num); EXPECT_NE(ds, nullptr); - // Create a Rename operation on ds - ds = ds->Rename({"image", "label"}, {"col1", "col2"}); + // Create objects for the tensor ops + std::shared_ptr resize_op = vision::Resize({30, 30}); + EXPECT_NE(resize_op, nullptr); + + std::shared_ptr center_crop_op = vision::CenterCrop({16, 16}); + EXPECT_NE(center_crop_op, nullptr); + + // Create a Map operation on ds + ds = ds->Map({resize_op, center_crop_op}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -677,66 +658,97 @@ TEST_F(MindDataTestPipeline, TestRenameSuccess) { iter->GetNextRow(&row); uint64_t i = 0; - EXPECT_NE(row.find("col1"), row.end()); - EXPECT_NE(row.find("col2"), row.end()); - EXPECT_EQ(row.find("image"), row.end()); - EXPECT_EQ(row.find("label"), row.end()); - while (row.size() != 0) { i++; - auto image = row["col1"]; + auto image = row["image"]; MS_LOG(INFO) << "Tensor image shape: " << image->shape(); iter->GetNextRow(&row); } - EXPECT_EQ(i, 20); + EXPECT_EQ(i, 40); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestRenameFail) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRenameFail."; - // We expect this test to fail because input and output in Rename are not the same size + +TEST_F(MindDataTestPipeline, TestZipFail) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipFail."; + // We expect this test to fail because we are the both datasets we are zipping have "image" and "label" columns + // and zip doesn't accept datasets with same column names // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - // Create a Repeat operation on ds - int32_t repeat_num = 2; - ds = ds->Repeat(repeat_num); + // Create an ImageFolder Dataset + std::shared_ptr ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10)); + EXPECT_NE(ds1, nullptr); + + // Create a Zip operation on the datasets + ds = Zip({ds, ds1}); EXPECT_NE(ds, nullptr); - // Create a Rename operation on ds - ds = ds->Rename({"image", "label"}, {"col2"}); + // Create a Batch operation on ds + int32_t batch_size = 1; + ds = ds->Batch(batch_size); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_EQ(iter, nullptr); +} + +TEST_F(MindDataTestPipeline, TestZipFail2) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipFail2."; + // This case is expected to fail because the input dataset is empty. + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + + // Create a Zip operation on the datasets + // Input dataset to zip is empty + ds = Zip({}); EXPECT_EQ(ds, nullptr); } -TEST_F(MindDataTestPipeline, TestConcatSuccess) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatSuccess."; +TEST_F(MindDataTestPipeline, TestZipSuccess) { + // Testing the member zip() function + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipSuccess."; // Create an ImageFolder Dataset - // Column names: {"image", "label"} std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - // Create a Cifar10 Dataset - // Column names: {"image", "label"} + // Create a Project operation on ds + std::vector column_project = {"image"}; + ds = ds->Project(column_project); + EXPECT_NE(ds, nullptr); + + // Create an ImageFolder Dataset + std::shared_ptr ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10)); + EXPECT_NE(ds1, nullptr); + + // Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names) + ds1 = ds1->Rename({"image", "label"}, {"col1", "col2"}); + EXPECT_NE(ds1, nullptr); + folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds2 = Cifar10(folder_path, RandomSampler(false, 9)); + std::shared_ptr ds2 = Cifar10(folder_path, RandomSampler(false, 10)); EXPECT_NE(ds2, nullptr); // Create a Project operation on ds - ds = ds->Project({"image"}); - EXPECT_NE(ds, nullptr); - ds2 = ds2->Project({"image"}); - EXPECT_NE(ds, nullptr); + column_project = {"label"}; + ds2 = ds2->Project(column_project); + EXPECT_NE(ds2, nullptr); - // Create a Concat operation on the ds - ds = ds->Concat({ds2}); + // Create a Zip operation on the datasets + ds = ds->Zip({ds1, ds2}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -752,6 +764,14 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess) { // Iterate the dataset and get each row std::unordered_map> row; iter->GetNextRow(&row); + + // Check zipped column names + EXPECT_EQ(row.size(), 4); + EXPECT_NE(row.find("image"), row.end()); + EXPECT_NE(row.find("label"), row.end()); + EXPECT_NE(row.find("col1"), row.end()); + EXPECT_NE(row.find("col2"), row.end()); + uint64_t i = 0; while (row.size() != 0) { i++; @@ -760,35 +780,29 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess) { iter->GetNextRow(&row); } - EXPECT_EQ(i, 19); + EXPECT_EQ(i, 10); + // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestConcatSuccess2) { - // Test "+" operator to concat two datasets - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatSuccess2."; +TEST_F(MindDataTestPipeline, TestZipSuccess2) { + // Testing the static zip() function + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipSuccess2."; // Create an ImageFolder Dataset - // Column names: {"image", "label"} std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 9)); EXPECT_NE(ds, nullptr); - - // Create a Cifar10 Dataset - // Column names: {"image", "label"} - folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds2 = Cifar10(folder_path, RandomSampler(false, 9)); + std::shared_ptr ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds2, nullptr); - // Create a Project operation on ds - ds = ds->Project({"image"}); - EXPECT_NE(ds, nullptr); - ds2 = ds2->Project({"image"}); + // Create a Rename operation on ds (so that the 2 datasets we are going to zip have distinct column names) + ds = ds->Rename({"image", "label"}, {"col1", "col2"}); EXPECT_NE(ds, nullptr); - // Create a Concat operation on the ds - ds = ds + ds2; + // Create a Zip operation on the datasets + ds = Zip({ds, ds2}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -804,6 +818,14 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess2) { // Iterate the dataset and get each row std::unordered_map> row; iter->GetNextRow(&row); + + // Check zipped column names + EXPECT_EQ(row.size(), 4); + EXPECT_NE(row.find("image"), row.end()); + EXPECT_NE(row.find("label"), row.end()); + EXPECT_NE(row.find("col1"), row.end()); + EXPECT_NE(row.find("col2"), row.end()); + uint64_t i = 0; while (row.size() != 0) { i++; @@ -812,54 +834,8 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess2) { iter->GetNextRow(&row); } - EXPECT_EQ(i, 19); + EXPECT_EQ(i, 9); + // Manually terminate the pipeline iter->Stop(); } - -TEST_F(MindDataTestPipeline, TestConcatFail1) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatFail1."; - // This case is expected to fail because the input column names of concatenated datasets are not the same - - // Create an ImageFolder Dataset - // Column names: {"image", "label"} - std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_NE(ds, nullptr); - std::shared_ptr ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_NE(ds, nullptr); - - // Create a Rename operation on ds - ds2 = ds2->Rename({"image", "label"}, {"col1", "col2"}); - EXPECT_NE(ds, nullptr); - - // Create a Project operation on the ds - // Name of datasets to concat doesn't not match - ds = ds->Concat({ds2}); - EXPECT_NE(ds, nullptr); - - // Create a Batch operation on ds - int32_t batch_size = 1; - ds = ds->Batch(batch_size); - EXPECT_NE(ds, nullptr); - - // Create an iterator over the result of the above dataset - // This will trigger the creation of the Execution Tree and launch it. - std::shared_ptr iter = ds->CreateIterator(); - EXPECT_EQ(iter, nullptr); -} - -TEST_F(MindDataTestPipeline, TestConcatFail2) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatFail2."; - // This case is expected to fail because the input dataset is empty. - - // Create an ImageFolder Dataset - std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_NE(ds, nullptr); - - // Create a Project operation on the ds - // Input dataset to concat is empty - ds = ds->Concat({}); - EXPECT_EQ(ds, nullptr); -} diff --git a/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc b/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc index 68c255574..b80f76f58 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc @@ -13,54 +13,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include -#include - -#include "utils/log_adapter.h" -#include "utils/ms_utils.h" #include "common/common.h" -#include "gtest/gtest.h" -#include "securec.h" -#include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/include/status.h" -#include "minddata/dataset/include/transforms.h" -#include "minddata/dataset/include/iterator.h" -#include "minddata/dataset/core/constants.h" -#include "minddata/dataset/core/tensor_shape.h" -#include "minddata/dataset/core/tensor.h" -#include "minddata/dataset/include/samplers.h" #include "minddata/dataset/engine/datasetops/source/voc_op.h" +#include "minddata/dataset/include/datasets.h" using namespace mindspore::dataset::api; -using mindspore::MsLogLevel::ERROR; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::LogStream; using mindspore::dataset::Tensor; using mindspore::dataset::TensorShape; -using mindspore::dataset::TensorImpl; using mindspore::dataset::DataType; -using mindspore::dataset::Status; -using mindspore::dataset::BorderType; -using mindspore::dataset::dsize_t; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; -TEST_F(MindDataTestPipeline, TestVOCSegmentation) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCSegmentation."; +TEST_F(MindDataTestPipeline, TestVOCClassIndex) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCClassIndex."; // Create a VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Segmentation", "train", {}, false, SequentialSampler(0, 3)); - EXPECT_NE(ds, nullptr); + std::map class_index; + class_index["car"] = 0; + class_index["cat"] = 1; + class_index["train"] = 9; - // Create a Repeat operation on ds - int32_t repeat_num = 2; - ds = ds->Repeat(repeat_num); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -72,23 +48,20 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) { std::unordered_map> row; iter->GetNextRow(&row); - // Check if VOCOp read correct images/targets - using Tensor = mindspore::dataset::Tensor; - std::string expect_file[] = {"32", "33", "39", "32", "33", "39"}; + // Check if VOCOp read correct labels + // When we provide class_index, label of ["car","cat","train"] become [0,1,9] + std::shared_ptr expect_label; + Tensor::CreateFromMemory(TensorShape({1, 1}), DataType(DataType::DE_UINT32), nullptr, &expect_label); + + uint32_t expect[] = {9, 9, 9, 1, 1, 0}; uint64_t i = 0; while (row.size() != 0) { auto image = row["image"]; - auto target = row["target"]; + auto label = row["label"]; MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - MS_LOG(INFO) << "Tensor target shape: " << target->shape(); - - std::shared_ptr expect_image; - Tensor::CreateFromFile(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg", &expect_image); - EXPECT_EQ(*image, *expect_image); - - std::shared_ptr expect_target; - Tensor::CreateFromFile(folder_path + "/SegmentationClass/" + expect_file[i] + ".png", &expect_target); - EXPECT_EQ(*target, *expect_target); + MS_LOG(INFO) << "Tensor label shape: " << label->shape(); + expect_label->SetItemAt({0, 0}, expect[i]); + EXPECT_EQ(*label, *expect_label); iter->GetNextRow(&row); i++; @@ -100,33 +73,6 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestVOCSegmentationError1) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCSegmentationError1."; - - // Create a VOC Dataset - std::map class_index; - class_index["car"] = 0; - std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Segmentation", "train", class_index, false, RandomSampler(false, 6)); - - // Expect nullptr for segmentation task with class_index - EXPECT_EQ(ds, nullptr); -} - -TEST_F(MindDataTestPipeline, TestVOCInvalidTaskOrMode) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCInvalidTaskOrMode."; - - // Create a VOC Dataset - std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds_1 = VOC(folder_path, "Classification", "train", {}, false, SequentialSampler(0, 3)); - // Expect nullptr for invalid task - EXPECT_EQ(ds_1, nullptr); - - std::shared_ptr ds_2 = VOC(folder_path, "Segmentation", "validation", {}, false, RandomSampler(false, 4)); - // Expect nullptr for invalid mode - EXPECT_EQ(ds_2, nullptr); -} - TEST_F(MindDataTestPipeline, TestVOCDetection) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCDetection."; @@ -173,17 +119,31 @@ TEST_F(MindDataTestPipeline, TestVOCDetection) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestVOCClassIndex) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCClassIndex."; +TEST_F(MindDataTestPipeline, TestVOCInvalidTaskOrMode) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCInvalidTaskOrMode."; // Create a VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::map class_index; - class_index["car"] = 0; - class_index["cat"] = 1; - class_index["train"] = 9; + std::shared_ptr ds_1 = VOC(folder_path, "Classification", "train", {}, false, SequentialSampler(0, 3)); + // Expect nullptr for invalid task + EXPECT_EQ(ds_1, nullptr); - std::shared_ptr ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6)); + std::shared_ptr ds_2 = VOC(folder_path, "Segmentation", "validation", {}, false, RandomSampler(false, 4)); + // Expect nullptr for invalid mode + EXPECT_EQ(ds_2, nullptr); +} + +TEST_F(MindDataTestPipeline, TestVOCSegmentation) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCSegmentation."; + + // Create a VOC Dataset + std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; + std::shared_ptr ds = VOC(folder_path, "Segmentation", "train", {}, false, SequentialSampler(0, 3)); + EXPECT_NE(ds, nullptr); + + // Create a Repeat operation on ds + int32_t repeat_num = 2; + ds = ds->Repeat(repeat_num); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -195,20 +155,23 @@ TEST_F(MindDataTestPipeline, TestVOCClassIndex) { std::unordered_map> row; iter->GetNextRow(&row); - // Check if VOCOp read correct labels - // When we provide class_index, label of ["car","cat","train"] become [0,1,9] - std::shared_ptr expect_label; - Tensor::CreateFromMemory(TensorShape({1, 1}), DataType(DataType::DE_UINT32), nullptr, &expect_label); - - uint32_t expect[] = {9, 9, 9, 1, 1, 0}; + // Check if VOCOp read correct images/targets + using Tensor = mindspore::dataset::Tensor; + std::string expect_file[] = {"32", "33", "39", "32", "33", "39"}; uint64_t i = 0; while (row.size() != 0) { auto image = row["image"]; - auto label = row["label"]; + auto target = row["target"]; MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - MS_LOG(INFO) << "Tensor label shape: " << label->shape(); - expect_label->SetItemAt({0, 0}, expect[i]); - EXPECT_EQ(*label, *expect_label); + MS_LOG(INFO) << "Tensor target shape: " << target->shape(); + + std::shared_ptr expect_image; + Tensor::CreateFromFile(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg", &expect_image); + EXPECT_EQ(*image, *expect_image); + + std::shared_ptr expect_target; + Tensor::CreateFromFile(folder_path + "/SegmentationClass/" + expect_file[i] + ".png", &expect_target); + EXPECT_EQ(*target, *expect_target); iter->GetNextRow(&row); i++; @@ -219,3 +182,16 @@ TEST_F(MindDataTestPipeline, TestVOCClassIndex) { // Manually terminate the pipeline iter->Stop(); } + +TEST_F(MindDataTestPipeline, TestVOCSegmentationError1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCSegmentationError1."; + + // Create a VOC Dataset + std::map class_index; + class_index["car"] = 0; + std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; + std::shared_ptr ds = VOC(folder_path, "Segmentation", "train", class_index, false, RandomSampler(false, 6)); + + // Expect nullptr for segmentation task with class_index + EXPECT_EQ(ds, nullptr); +} diff --git a/tests/ut/cpp/dataset/c_api_datasets_test.cc b/tests/ut/cpp/dataset/c_api_datasets_test.cc index 4b21e1f04..31acdc770 100644 --- a/tests/ut/cpp/dataset/c_api_datasets_test.cc +++ b/tests/ut/cpp/dataset/c_api_datasets_test.cc @@ -13,58 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include -#include - -#include "utils/log_adapter.h" -#include "utils/ms_utils.h" #include "common/common.h" -#include "gtest/gtest.h" -#include "securec.h" #include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/include/status.h" -#include "minddata/dataset/include/transforms.h" -#include "minddata/dataset/include/iterator.h" -#include "minddata/dataset/core/constants.h" -#include "minddata/dataset/core/tensor_shape.h" -#include "minddata/dataset/core/tensor.h" -#include "minddata/dataset/include/samplers.h" using namespace mindspore::dataset::api; -using mindspore::MsLogLevel::ERROR; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::LogStream; using mindspore::dataset::Tensor; using mindspore::dataset::TensorShape; -using mindspore::dataset::TensorImpl; -using mindspore::dataset::DataType; -using mindspore::dataset::Status; -using mindspore::dataset::BorderType; -using mindspore::dataset::dsize_t; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; -TEST_F(MindDataTestPipeline, TestMnistFail1) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMnistFail1."; - - // Create a Mnist Dataset - std::shared_ptr ds = Mnist("", RandomSampler(false, 10)); - EXPECT_EQ(ds, nullptr); -} - -TEST_F(MindDataTestPipeline, TestImageFolderFail1) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestImageFolderFail1."; - - // Create an ImageFolder Dataset - std::shared_ptr ds = ImageFolder("", true, nullptr); - EXPECT_EQ(ds, nullptr); -} - TEST_F(MindDataTestPipeline, TestCelebADataset) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCelebADataset."; @@ -158,3 +117,19 @@ TEST_F(MindDataTestPipeline, TestCelebAException) { std::shared_ptr ds1 = CelebA(folder_path, invalid_dataset_type); EXPECT_EQ(ds1, nullptr); } + +TEST_F(MindDataTestPipeline, TestImageFolderFail1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestImageFolderFail1."; + + // Create an ImageFolder Dataset + std::shared_ptr ds = ImageFolder("", true, nullptr); + EXPECT_EQ(ds, nullptr); +} + +TEST_F(MindDataTestPipeline, TestMnistFail1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMnistFail1."; + + // Create a Mnist Dataset + std::shared_ptr ds = Mnist("", RandomSampler(false, 10)); + EXPECT_EQ(ds, nullptr); +} diff --git a/tests/ut/cpp/dataset/c_api_samplers_test.cc b/tests/ut/cpp/dataset/c_api_samplers_test.cc index 78a81184f..16807661e 100644 --- a/tests/ut/cpp/dataset/c_api_samplers_test.cc +++ b/tests/ut/cpp/dataset/c_api_samplers_test.cc @@ -13,33 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include -#include - -#include "utils/log_adapter.h" -#include "utils/ms_utils.h" #include "common/common.h" -#include "gtest/gtest.h" -#include "securec.h" #include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/include/status.h" -#include "minddata/dataset/include/transforms.h" -#include "minddata/dataset/include/iterator.h" -#include "minddata/dataset/core/constants.h" -#include "minddata/dataset/core/tensor_shape.h" -#include "minddata/dataset/core/tensor.h" -#include "minddata/dataset/include/samplers.h" using namespace mindspore::dataset::api; -using mindspore::MsLogLevel::ERROR; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::LogStream; using mindspore::dataset::Tensor; -using mindspore::dataset::Status; -using mindspore::dataset::BorderType; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_transforms_test.cc b/tests/ut/cpp/dataset/c_api_transforms_test.cc index e8c888f2c..8eae5ac45 100644 --- a/tests/ut/cpp/dataset/c_api_transforms_test.cc +++ b/tests/ut/cpp/dataset/c_api_transforms_test.cc @@ -13,64 +13,43 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include -#include - -#include "utils/log_adapter.h" -#include "utils/ms_utils.h" #include "common/common.h" -#include "gtest/gtest.h" -#include "securec.h" #include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/include/status.h" #include "minddata/dataset/include/transforms.h" -#include "minddata/dataset/include/iterator.h" -#include "minddata/dataset/core/constants.h" -#include "minddata/dataset/core/tensor_shape.h" -#include "minddata/dataset/core/tensor.h" -#include "minddata/dataset/include/samplers.h" using namespace mindspore::dataset::api; -using mindspore::LogStream; using mindspore::dataset::BorderType; -using mindspore::dataset::Status; using mindspore::dataset::Tensor; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::MsLogLevel::ERROR; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; -TEST_F(MindDataTestPipeline, TestUniformAugWithOps) { - // Create a Mnist Dataset - std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, RandomSampler(false, 20)); +TEST_F(MindDataTestPipeline, TestCutOut) { + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds - int32_t repeat_num = 1; + int32_t repeat_num = 2; ds = ds->Repeat(repeat_num); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops - std::shared_ptr resize_op = vision::Resize({30, 30}); - EXPECT_NE(resize_op, nullptr); - - std::shared_ptr random_crop_op = vision::RandomCrop({28, 28}); - EXPECT_NE(random_crop_op, nullptr); - - std::shared_ptr center_crop_op = vision::CenterCrop({16, 16}); - EXPECT_NE(center_crop_op, nullptr); + std::shared_ptr cut_out1 = vision::CutOut(30, 5); + EXPECT_NE(cut_out1, nullptr); - std::shared_ptr uniform_aug_op = vision::UniformAugment({random_crop_op, center_crop_op}, 2); - EXPECT_NE(uniform_aug_op, nullptr); + std::shared_ptr cut_out2 = vision::CutOut(30); + EXPECT_NE(cut_out2, nullptr); // Create a Map operation on ds - ds = ds->Map({resize_op, uniform_aug_op}); + ds = ds->Map({cut_out1, cut_out2}); + EXPECT_NE(ds, nullptr); + + // Create a Batch operation on ds + int32_t batch_size = 1; + ds = ds->Batch(batch_size); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -84,10 +63,10 @@ TEST_F(MindDataTestPipeline, TestUniformAugWithOps) { uint64_t i = 0; while (row.size() != 0) { - i++; - auto image = row["image"]; - MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - iter->GetNextRow(&row); + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + iter->GetNextRow(&row); } EXPECT_EQ(i, 20); @@ -96,10 +75,10 @@ TEST_F(MindDataTestPipeline, TestUniformAugWithOps) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestRandomFlip) { +TEST_F(MindDataTestPipeline, TestDecode) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -108,14 +87,11 @@ TEST_F(MindDataTestPipeline, TestRandomFlip) { EXPECT_NE(ds, nullptr); // Create objects for the tensor ops - std::shared_ptr random_vertical_flip_op = vision::RandomVerticalFlip(0.5); - EXPECT_NE(random_vertical_flip_op, nullptr); - - std::shared_ptr random_horizontal_flip_op = vision::RandomHorizontalFlip(0.5); - EXPECT_NE(random_horizontal_flip_op, nullptr); + std::shared_ptr decode = vision::Decode(true); + EXPECT_NE(decode, nullptr); // Create a Map operation on ds - ds = ds->Map({random_vertical_flip_op, random_horizontal_flip_op}); + ds = ds->Map({decode}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -139,37 +115,34 @@ TEST_F(MindDataTestPipeline, TestRandomFlip) { MS_LOG(INFO) << "Tensor image shape: " << image->shape(); iter->GetNextRow(&row); } - EXPECT_EQ(i, 20); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess1) { - // Create a Cifar10 Dataset - std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, RandomSampler(false, 10)); +TEST_F(MindDataTestPipeline, TestHwcToChw) { + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); - // Create a Batch operation on ds - int32_t batch_size = 5; - ds = ds->Batch(batch_size); + // Create a Repeat operation on ds + int32_t repeat_num = 2; + ds = ds->Repeat(repeat_num); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops - std::shared_ptr one_hot_op = vision::OneHot(10); - EXPECT_NE(one_hot_op, nullptr); + std::shared_ptr channel_swap = vision::HWC2CHW(); + EXPECT_NE(channel_swap, nullptr); // Create a Map operation on ds - ds = ds->Map({one_hot_op},{"label"}); + ds = ds->Map({channel_swap}); EXPECT_NE(ds, nullptr); - std::shared_ptr mixup_batch_op = vision::MixUpBatch(0.5); - EXPECT_NE(mixup_batch_op, nullptr); - - // Create a Map operation on ds - ds = ds->Map({mixup_batch_op}, {"image", "label"}); + // Create a Batch operation on ds + int32_t batch_size = 1; + ds = ds->Batch(batch_size); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -186,16 +159,18 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess1) { i++; auto image = row["image"]; MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + // check if the image is in NCHW + EXPECT_EQ(batch_size == image->shape()[0] && 3 == image->shape()[1] + && 2268 == image->shape()[2] && 4032 == image->shape()[3], true); iter->GetNextRow(&row); } - - EXPECT_EQ(i, 2); + EXPECT_EQ(i, 20); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) { +TEST_F(MindDataTestPipeline, TestMixUpBatchFail1) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::shared_ptr ds = Cifar10(folder_path, RandomSampler(false, 10)); @@ -214,7 +189,30 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) { ds = ds->Map({one_hot_op},{"label"}); EXPECT_NE(ds, nullptr); - std::shared_ptr mixup_batch_op = vision::MixUpBatch(); + std::shared_ptr mixup_batch_op = vision::MixUpBatch(-1); + EXPECT_EQ(mixup_batch_op, nullptr); +} + +TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess1) { + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + + // Create a Batch operation on ds + int32_t batch_size = 5; + ds = ds->Batch(batch_size); + EXPECT_NE(ds, nullptr); + + // Create objects for the tensor ops + std::shared_ptr one_hot_op = vision::OneHot(10); + EXPECT_NE(one_hot_op, nullptr); + + // Create a Map operation on ds + ds = ds->Map({one_hot_op},{"label"}); + EXPECT_NE(ds, nullptr); + + std::shared_ptr mixup_batch_op = vision::MixUpBatch(0.5); EXPECT_NE(mixup_batch_op, nullptr); // Create a Map operation on ds @@ -244,7 +242,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestMixUpBatchFail1) { +TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; std::shared_ptr ds = Cifar10(folder_path, RandomSampler(false, 10)); @@ -263,38 +261,11 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchFail1) { ds = ds->Map({one_hot_op},{"label"}); EXPECT_NE(ds, nullptr); - std::shared_ptr mixup_batch_op = vision::MixUpBatch(-1); - EXPECT_EQ(mixup_batch_op, nullptr); -} - -TEST_F(MindDataTestPipeline, TestPad) { - // Create an ImageFolder Dataset - std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); - EXPECT_NE(ds, nullptr); - - // Create a Repeat operation on ds - int32_t repeat_num = 2; - ds = ds->Repeat(repeat_num); - EXPECT_NE(ds, nullptr); - - // Create objects for the tensor ops - std::shared_ptr pad_op1 = vision::Pad({1, 2, 3, 4}, {0}, BorderType::kSymmetric); - EXPECT_NE(pad_op1, nullptr); - - std::shared_ptr pad_op2 = vision::Pad({1}, {1, 1, 1}, BorderType::kEdge); - EXPECT_NE(pad_op2, nullptr); - - std::shared_ptr pad_op3 = vision::Pad({1, 4}); - EXPECT_NE(pad_op3, nullptr); + std::shared_ptr mixup_batch_op = vision::MixUpBatch(); + EXPECT_NE(mixup_batch_op, nullptr); // Create a Map operation on ds - ds = ds->Map({pad_op1, pad_op2, pad_op3}); - EXPECT_NE(ds, nullptr); - - // Create a Batch operation on ds - int32_t batch_size = 1; - ds = ds->Batch(batch_size); + ds = ds->Map({mixup_batch_op}, {"image", "label"}); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -314,13 +285,13 @@ TEST_F(MindDataTestPipeline, TestPad) { iter->GetNextRow(&row); } - EXPECT_EQ(i, 20); + EXPECT_EQ(i, 2); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestCutOut) { +TEST_F(MindDataTestPipeline, TestNormalize) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); @@ -332,14 +303,11 @@ TEST_F(MindDataTestPipeline, TestCutOut) { EXPECT_NE(ds, nullptr); // Create objects for the tensor ops - std::shared_ptr cut_out1 = vision::CutOut(30, 5); - EXPECT_NE(cut_out1, nullptr); - - std::shared_ptr cut_out2 = vision::CutOut(30); - EXPECT_NE(cut_out2, nullptr); + std::shared_ptr normalize = vision::Normalize({121.0, 115.0, 100.0}, {70.0, 68.0, 71.0}); + EXPECT_NE(normalize, nullptr); // Create a Map operation on ds - ds = ds->Map({cut_out1, cut_out2}); + ds = ds->Map({normalize}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -370,7 +338,7 @@ TEST_F(MindDataTestPipeline, TestCutOut) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestNormalize) { +TEST_F(MindDataTestPipeline, TestPad) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); @@ -382,11 +350,17 @@ TEST_F(MindDataTestPipeline, TestNormalize) { EXPECT_NE(ds, nullptr); // Create objects for the tensor ops - std::shared_ptr normalize = vision::Normalize({121.0, 115.0, 100.0}, {70.0, 68.0, 71.0}); - EXPECT_NE(normalize, nullptr); + std::shared_ptr pad_op1 = vision::Pad({1, 2, 3, 4}, {0}, BorderType::kSymmetric); + EXPECT_NE(pad_op1, nullptr); + + std::shared_ptr pad_op2 = vision::Pad({1}, {1, 1, 1}, BorderType::kEdge); + EXPECT_NE(pad_op2, nullptr); + + std::shared_ptr pad_op3 = vision::Pad({1, 4}); + EXPECT_NE(pad_op3, nullptr); // Create a Map operation on ds - ds = ds->Map({normalize}); + ds = ds->Map({pad_op1, pad_op2, pad_op3}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -405,10 +379,10 @@ TEST_F(MindDataTestPipeline, TestNormalize) { uint64_t i = 0; while (row.size() != 0) { - i++; - auto image = row["image"]; - MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - iter->GetNextRow(&row); + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + iter->GetNextRow(&row); } EXPECT_EQ(i, 20); @@ -417,10 +391,26 @@ TEST_F(MindDataTestPipeline, TestNormalize) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestDecode) { +TEST_F(MindDataTestPipeline, TestRandomAffineFail) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomAffineFail with invalid params."; + + // Create objects for the tensor ops + std::shared_ptr affine = vision::RandomAffine({0.0, 0.0}, {}); + EXPECT_EQ(affine, nullptr); + // Invalid number of values for translate + affine = vision::RandomAffine({0.0, 0.0}, {1, 1, 1, 1}); + EXPECT_EQ(affine, nullptr); + // Invalid number of values for shear + affine = vision::RandomAffine({30.0, 30.0}, {0.0, 0.0}, {2.0, 2.0}, {10.0, 10.0}); + EXPECT_EQ(affine, nullptr); +} + +TEST_F(MindDataTestPipeline, TestRandomAffineSuccess1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomAffineSuccess1 with non-default params."; + // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -429,11 +419,12 @@ TEST_F(MindDataTestPipeline, TestDecode) { EXPECT_NE(ds, nullptr); // Create objects for the tensor ops - std::shared_ptr decode = vision::Decode(true); - EXPECT_NE(decode, nullptr); + std::shared_ptr affine = + vision::RandomAffine({30.0, 30.0}, {0.0, 0.0}, {2.0, 2.0}, {10.0, 10.0, 20.0, 20.0}); + EXPECT_NE(affine, nullptr); // Create a Map operation on ds - ds = ds->Map({decode}); + ds = ds->Map({affine}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -457,13 +448,16 @@ TEST_F(MindDataTestPipeline, TestDecode) { MS_LOG(INFO) << "Tensor image shape: " << image->shape(); iter->GetNextRow(&row); } + EXPECT_EQ(i, 20); // Manually terminate the pipeline iter->Stop(); } -TEST_F(MindDataTestPipeline, TestHwcToChw) { +TEST_F(MindDataTestPipeline, TestRandomAffineSuccess2) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomAffineSuccess2 with default params."; + // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); @@ -475,11 +469,11 @@ TEST_F(MindDataTestPipeline, TestHwcToChw) { EXPECT_NE(ds, nullptr); // Create objects for the tensor ops - std::shared_ptr channel_swap = vision::HWC2CHW(); - EXPECT_NE(channel_swap, nullptr); + std::shared_ptr affine = vision::RandomAffine({0.0, 0.0}); + EXPECT_NE(affine, nullptr); // Create a Map operation on ds - ds = ds->Map({channel_swap}); + ds = ds->Map({affine}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -501,11 +495,9 @@ TEST_F(MindDataTestPipeline, TestHwcToChw) { i++; auto image = row["image"]; MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - // check if the image is in NCHW - EXPECT_EQ(batch_size == image->shape()[0] && 3 == image->shape()[1] - && 2268 == image->shape()[2] && 4032 == image->shape()[3], true); iter->GetNextRow(&row); } + EXPECT_EQ(i, 20); // Manually terminate the pipeline @@ -628,9 +620,7 @@ TEST_F(MindDataTestPipeline, TestRandomSharpness) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestRandomAffineSuccess1) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomAffineSuccess1 with non-default params."; - +TEST_F(MindDataTestPipeline, TestRandomFlip) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); @@ -642,12 +632,14 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess1) { EXPECT_NE(ds, nullptr); // Create objects for the tensor ops - std::shared_ptr affine = - vision::RandomAffine({30.0, 30.0}, {0.0, 0.0}, {2.0, 2.0}, {10.0, 10.0, 20.0, 20.0}); - EXPECT_NE(affine, nullptr); + std::shared_ptr random_vertical_flip_op = vision::RandomVerticalFlip(0.5); + EXPECT_NE(random_vertical_flip_op, nullptr); + + std::shared_ptr random_horizontal_flip_op = vision::RandomHorizontalFlip(0.5); + EXPECT_NE(random_horizontal_flip_op, nullptr); // Create a Map operation on ds - ds = ds->Map({affine}); + ds = ds->Map({random_vertical_flip_op, random_horizontal_flip_op}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -678,9 +670,7 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess1) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestRandomAffineSuccess2) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomAffineSuccess2 with default params."; - +TEST_F(MindDataTestPipeline, TestRandomRotation) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); @@ -692,11 +682,11 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess2) { EXPECT_NE(ds, nullptr); // Create objects for the tensor ops - std::shared_ptr affine = vision::RandomAffine({0.0, 0.0}); - EXPECT_NE(affine, nullptr); + std::shared_ptr random_rotation_op = vision::RandomRotation({-180, 180}); + EXPECT_NE(random_rotation_op, nullptr); // Create a Map operation on ds - ds = ds->Map({affine}); + ds = ds->Map({random_rotation_op}); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -727,42 +717,32 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess2) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestRandomAffineFail) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomAffineFail with invalid params."; - - // Create objects for the tensor ops - std::shared_ptr affine = vision::RandomAffine({0.0, 0.0}, {}); - EXPECT_EQ(affine, nullptr); - // Invalid number of values for translate - affine = vision::RandomAffine({0.0, 0.0}, {1, 1, 1, 1}); - EXPECT_EQ(affine, nullptr); - // Invalid number of values for shear - affine = vision::RandomAffine({30.0, 30.0}, {0.0, 0.0}, {2.0, 2.0}, {10.0, 10.0}); - EXPECT_EQ(affine, nullptr); -} - -TEST_F(MindDataTestPipeline, TestRandomRotation) { - // Create an ImageFolder Dataset - std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); +TEST_F(MindDataTestPipeline, TestUniformAugWithOps) { + // Create a Mnist Dataset + std::string folder_path = datasets_root_path_ + "/testMnistData/"; + std::shared_ptr ds = Mnist(folder_path, RandomSampler(false, 20)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds - int32_t repeat_num = 2; + int32_t repeat_num = 1; ds = ds->Repeat(repeat_num); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops - std::shared_ptr random_rotation_op = vision::RandomRotation({-180, 180}); - EXPECT_NE(random_rotation_op, nullptr); + std::shared_ptr resize_op = vision::Resize({30, 30}); + EXPECT_NE(resize_op, nullptr); - // Create a Map operation on ds - ds = ds->Map({random_rotation_op}); - EXPECT_NE(ds, nullptr); + std::shared_ptr random_crop_op = vision::RandomCrop({28, 28}); + EXPECT_NE(random_crop_op, nullptr); - // Create a Batch operation on ds - int32_t batch_size = 1; - ds = ds->Batch(batch_size); + std::shared_ptr center_crop_op = vision::CenterCrop({16, 16}); + EXPECT_NE(center_crop_op, nullptr); + + std::shared_ptr uniform_aug_op = vision::UniformAugment({random_crop_op, center_crop_op}, 2); + EXPECT_NE(uniform_aug_op, nullptr); + + // Create a Map operation on ds + ds = ds->Map({resize_op, uniform_aug_op}); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset -- GitLab