Skip to content

M
mindspore

正统之独孤求败 / mindspore
与 Fork 源项目一致

Fork自 MindSpore / mindspore

通知 1
Star 0
Fork 0
M
mindspore

体验新版 GitCode，发现更多精彩内容 >>

提交 deb11464 编写于 作者: T tony_liu2
浏览文件
操作

reorganize headers and tests 

remove headers

fix iterator file

fix pr comments

fix new file
上级 64b0feb7
隐藏空白更改
内联 并排
Showing with 979 addition and 1196 deletion
tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc
浏览文件 @ deb11464
......@@ -13,37 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fstream>
#include <iostream>
#include <memory>
#include <vector>
#include <string>
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "securec.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/status.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/iterator.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/include/samplers.h"
using namespace mindspore::dataset::api;
using mindspore::MsLogLevel::ERROR;
using mindspore::ExceptionType::NoExceptionType;
using mindspore::LogStream;
using mindspore::dataset::Tensor;
using mindspore::dataset::TensorShape;
using mindspore::dataset::TensorImpl;
using mindspore::dataset::DataType;
using mindspore::dataset::Status;
using mindspore::dataset::BorderType;
using mindspore::dataset::dsize_t;
class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
......@@ -83,14 +57,6 @@ TEST_F(MindDataTestPipeline, TestCifar10Dataset) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCifar10DatasetFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10DatasetFail1.";
// Create a Cifar10 Dataset
std::shared_ptr<Dataset> ds = Cifar10("", RandomSampler(false, 10));
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestCifar100Dataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar100Dataset.";
......@@ -133,3 +99,11 @@ TEST_F(MindDataTestPipeline, TestCifar100DatasetFail1) {
std::shared_ptr<Dataset> ds = Cifar100("", RandomSampler(false, 10));
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestCifar10DatasetFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10DatasetFail1.";
// Create a Cifar10 Dataset
std::shared_ptr<Dataset> ds = Cifar10("", RandomSampler(false, 10));
EXPECT_EQ(ds, nullptr);
}
tests/ut/cpp/dataset/c_api_dataset_clue_test.cc
浏览文件 @ deb11464
......@@ -14,9 +14,9 @@
* limitations under the License.
*/
#include "common/common.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/include/datasets.h"
using namespace mindspore::dataset::api;
using mindspore::dataset::ShuffleMode;
......@@ -27,76 +27,6 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
TEST_F(MindDataTestPipeline, TestCLUEDatasetBasic) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetBasic.";
// Create a CLUEFile Dataset, with single CLUE file
std::string clue_file = datasets_root_path_ + "/testCLUE/afqmc/train.json";
std::string task = "AFQMC";
std::string usage = "train";
std::shared_ptr<Dataset> ds = CLUE({clue_file}, task, usage, 2);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
EXPECT_NE(row.find("sentence1"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
auto text = row["sentence1"];
MS_LOG(INFO) << "Tensor text shape: " << text->shape();
i++;
iter->GetNextRow(&row);
}
// Expect 2 samples
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCLUEDatasetDistribution) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetDistribution.";
// Create a CLUEFile Dataset, with single CLUE file
std::string clue_file = datasets_root_path_ + "/testCLUE/afqmc/train.json";
std::string task = "AFQMC";
std::string usage = "train";
std::shared_ptr<Dataset> ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, 3, 0);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
EXPECT_NE(row.find("sentence1"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
auto text = row["sentence1"];
MS_LOG(INFO) << "Tensor text shape: " << text->shape();
i++;
iter->GetNextRow(&row);
}
// Expect 1 samples
EXPECT_EQ(i, 1);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCLUEDatasetAFQMC) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetAFQMC.";
......@@ -194,6 +124,41 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetAFQMC) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCLUEDatasetBasic) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetBasic.";
// Create a CLUEFile Dataset, with single CLUE file
std::string clue_file = datasets_root_path_ + "/testCLUE/afqmc/train.json";
std::string task = "AFQMC";
std::string usage = "train";
std::shared_ptr<Dataset> ds = CLUE({clue_file}, task, usage, 2);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
EXPECT_NE(row.find("sentence1"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
auto text = row["sentence1"];
MS_LOG(INFO) << "Tensor text shape: " << text->shape();
i++;
iter->GetNextRow(&row);
}
// Expect 2 samples
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCLUEDatasetCMNLI) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetCMNLI.";
......@@ -284,14 +249,14 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetCSL) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCLUEDatasetIFLYTEK) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetIFLYTEK.";
TEST_F(MindDataTestPipeline, TestCLUEDatasetDistribution) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetDistribution.";
// Create a CLUEFile Dataset, with single CLUE file
std::string clue_file = datasets_root_path_ + "/testCLUE/iflytek/train.json";
std::string task = "IFLYTEK";
std::string clue_file = datasets_root_path_ + "/testCLUE/afqmc/train.json";
std::string task = "AFQMC";
std::string usage = "train";
std::shared_ptr<Dataset> ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kFalse);
std::shared_ptr<Dataset> ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, 3, 0);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -303,38 +268,61 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetIFLYTEK) {
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
EXPECT_NE(row.find("sentence"), row.end());
std::vector<std::string> expected_result = {
"第一个文本",
"第二个文本",
"第三个文本"
};
EXPECT_NE(row.find("sentence1"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
auto text = row["sentence"];
std::string_view sv;
text->GetItemAt(&sv, {0});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
auto text = row["sentence1"];
MS_LOG(INFO) << "Tensor text shape: " << text->shape();
iter->GetNextRow(&row);
i++;
iter->GetNextRow(&row);
}
// Expect 3 samples
EXPECT_EQ(i, 3);
// Expect 1 samples
EXPECT_EQ(i, 1);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCLUEDatasetTNEWS) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetTNEWS.";
TEST_F(MindDataTestPipeline, TestCLUEDatasetException) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetException.";
// Create a CLUE Dataset
std::string clue_file = datasets_root_path_ + "/testCLUE/wsc/train.json";
std::string task = "WSC";
std::string usage = "train";
std::string invalid_clue_file = "./NotExistFile";
std::shared_ptr<Dataset> ds0 = CLUE({}, task, usage);
EXPECT_EQ(ds0, nullptr);
std::shared_ptr<Dataset> ds1 = CLUE({invalid_clue_file}, task, usage);
EXPECT_EQ(ds1, nullptr);
std::shared_ptr<Dataset> ds2 = CLUE({clue_file}, "invalid_task", usage);
EXPECT_EQ(ds2, nullptr);
std::shared_ptr<Dataset> ds3 = CLUE({clue_file}, task, "invalid_usage");
EXPECT_EQ(ds3, nullptr);
std::shared_ptr<Dataset> ds4 = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, 2, 2);
EXPECT_EQ(ds4, nullptr);
std::shared_ptr<Dataset> ds5 = CLUE({clue_file}, task, usage, -1, ShuffleMode::kGlobal);
EXPECT_EQ(ds5, nullptr);
std::shared_ptr<Dataset> ds6 = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, -1);
EXPECT_EQ(ds6, nullptr);
std::shared_ptr<Dataset> ds7 = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, 0, -1);
EXPECT_EQ(ds7, nullptr);
}
TEST_F(MindDataTestPipeline, TestCLUEDatasetIFLYTEK) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetIFLYTEK.";
// Create a CLUEFile Dataset, with single CLUE file
std::string clue_file = datasets_root_path_ + "/testCLUE/tnews/train.json";
std::string task = "TNEWS";
std::string clue_file = datasets_root_path_ + "/testCLUE/iflytek/train.json";
std::string task = "IFLYTEK";
std::string usage = "train";
std::shared_ptr<Dataset> ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kFalse);
EXPECT_NE(ds, nullptr);
......@@ -350,9 +338,9 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetTNEWS) {
EXPECT_NE(row.find("sentence"), row.end());
std::vector<std::string> expected_result = {
"新闻1",
"新闻2",
"新闻3"
"第一个文本",
"第二个文本",
"第三个文本"
};
uint64_t i = 0;
......@@ -374,17 +362,31 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetTNEWS) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCLUEDatasetWSC) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetWSC.";
TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleFiles) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetShuffleFiles.";
// Test CLUE Dataset with files shuffle, num_parallel_workers=1
// Create a CLUEFile Dataset, with single CLUE file
std::string clue_file = datasets_root_path_ + "/testCLUE/wsc/train.json";
std::string task = "WSC";
// Set configuration
uint32_t original_seed = GlobalContext::config_manager()->seed();
uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers();
MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers;
GlobalContext::config_manager()->set_seed(135);
GlobalContext::config_manager()->set_num_parallel_workers(1);
// Create a CLUE Dataset, with two text files
// Note: train.json has 3 rows
// Note: dev.json has 3 rows
// Use default of all samples
// They have the same keywords
// Set shuffle to files shuffle
std::string clue_file1 = datasets_root_path_ + "/testCLUE/afqmc/train.json";
std::string clue_file2 = datasets_root_path_ + "/testCLUE/afqmc/dev.json";
std::string task = "AFQMC";
std::string usage = "train";
std::shared_ptr<Dataset> ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kFalse);
std::shared_ptr<Dataset> ds = CLUE({clue_file1, clue_file2}, task, usage, 0, ShuffleMode::kFiles);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Create an iterator over the result of the above dataset.
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
......@@ -393,30 +395,37 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetWSC) {
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
EXPECT_NE(row.find("text"), row.end());
EXPECT_NE(row.find("sentence1"), row.end());
std::vector<std::string> expected_result = {
"小明呢，他在哪？",
"小红刚刚看到小明，他在操场",
"等小明回来，小张你叫他交作业"
"蚂蚁借呗等额还款能否换成先息后本",
"蚂蚁花呗说我违约了",
"帮我看看本月花呗账单结清了没",
"你有花呗吗",
"吃饭能用花呗吗",
"蚂蚁花呗支付金额有什么限制"
};
uint64_t i = 0;
while (row.size() != 0) {
auto text = row["text"];
auto text = row["sentence1"];
std::string_view sv;
text->GetItemAt(&sv, {0});
std::string ss(sv);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
MS_LOG(INFO) << "Tensor text shape: " << text->shape();
iter->GetNextRow(&row);
i++;
iter->GetNextRow(&row);
}
// Expect 3 samples
EXPECT_EQ(i, 3);
// Expect 3 + 3 = 6 samples
EXPECT_EQ(i, 6);
// Manually terminate the pipeline
iter->Stop();
// Restore configuration
GlobalContext::config_manager()->set_seed(original_seed);
GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers);
}
TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleGlobal) {
......@@ -476,31 +485,17 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleGlobal) {
GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers);
}
TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleFiles) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetShuffleFiles.";
// Test CLUE Dataset with files shuffle, num_parallel_workers=1
// Set configuration
uint32_t original_seed = GlobalContext::config_manager()->seed();
uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers();
MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers;
GlobalContext::config_manager()->set_seed(135);
GlobalContext::config_manager()->set_num_parallel_workers(1);
TEST_F(MindDataTestPipeline, TestCLUEDatasetTNEWS) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetTNEWS.";
// Create a CLUE Dataset, with two text files
// Note: train.json has 3 rows
// Note: dev.json has 3 rows
// Use default of all samples
// They have the same keywords
// Set shuffle to files shuffle
std::string clue_file1 = datasets_root_path_ + "/testCLUE/afqmc/train.json";
std::string clue_file2 = datasets_root_path_ + "/testCLUE/afqmc/dev.json";
std::string task = "AFQMC";
// Create a CLUEFile Dataset, with single CLUE file
std::string clue_file = datasets_root_path_ + "/testCLUE/tnews/train.json";
std::string task = "TNEWS";
std::string usage = "train";
std::shared_ptr<Dataset> ds = CLUE({clue_file1, clue_file2}, task, usage, 0, ShuffleMode::kFiles);
std::shared_ptr<Dataset> ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kFalse);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset.
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
......@@ -509,68 +504,73 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleFiles) {
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
EXPECT_NE(row.find("sentence1"), row.end());
EXPECT_NE(row.find("sentence"), row.end());
std::vector<std::string> expected_result = {
"蚂蚁借呗等额还款能否换成先息后本",
"蚂蚁花呗说我违约了",
"帮我看看本月花呗账单结清了没",
"你有花呗吗",
"吃饭能用花呗吗",
"蚂蚁花呗支付金额有什么限制"
"新闻1",
"新闻2",
"新闻3"
};
uint64_t i = 0;
while (row.size() != 0) {
auto text = row["sentence1"];
auto text = row["sentence"];
std::string_view sv;
text->GetItemAt(&sv, {0});
std::string ss(sv);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
i++;
MS_LOG(INFO) << "Tensor text shape: " << text->shape();
iter->GetNextRow(&row);
i++;
}
// Expect 3 + 3 = 6 samples
EXPECT_EQ(i, 6);
// Expect 3 samples
EXPECT_EQ(i, 3);
// Manually terminate the pipeline
iter->Stop();
// Restore configuration
GlobalContext::config_manager()->set_seed(original_seed);
GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers);
}
TEST_F(MindDataTestPipeline, TestCLUEDatasetException) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetException.";
// Create a CLUE Dataset
TEST_F(MindDataTestPipeline, TestCLUEDatasetWSC) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCLUEDatasetWSC.";
// Create a CLUEFile Dataset, with single CLUE file
std::string clue_file = datasets_root_path_ + "/testCLUE/wsc/train.json";
std::string task = "WSC";
std::string usage = "train";
std::string invalid_clue_file = "./NotExistFile";
std::shared_ptr<Dataset> ds0 = CLUE({}, task, usage);
EXPECT_EQ(ds0, nullptr);
std::shared_ptr<Dataset> ds1 = CLUE({invalid_clue_file}, task, usage);
EXPECT_EQ(ds1, nullptr);
std::shared_ptr<Dataset> ds = CLUE({clue_file}, task, usage, 0, ShuffleMode::kFalse);
EXPECT_NE(ds, nullptr);
std::shared_ptr<Dataset> ds2 = CLUE({clue_file}, "invalid_task", usage);
EXPECT_EQ(ds2, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
std::shared_ptr<Dataset> ds3 = CLUE({clue_file}, task, "invalid_usage");
EXPECT_EQ(ds3, nullptr);
// Iterate the dataset and get each row
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
std::shared_ptr<Dataset> ds4 = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, 2, 2);
EXPECT_EQ(ds4, nullptr);
EXPECT_NE(row.find("text"), row.end());
std::vector<std::string> expected_result = {
"小明呢，他在哪？",
"小红刚刚看到小明，他在操场",
"等小明回来，小张你叫他交作业"
};
std::shared_ptr<Dataset> ds5 = CLUE({clue_file}, task, usage, -1, ShuffleMode::kGlobal);
EXPECT_EQ(ds5, nullptr);
uint64_t i = 0;
while (row.size() != 0) {
auto text = row["text"];
std::string_view sv;
text->GetItemAt(&sv, {0});
std::string ss(sv);
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
MS_LOG(INFO) << "Tensor text shape: " << text->shape();
iter->GetNextRow(&row);
i++;
}
std::shared_ptr<Dataset> ds6 = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, -1);
EXPECT_EQ(ds6, nullptr);
// Expect 3 samples
EXPECT_EQ(i, 3);
std::shared_ptr<Dataset> ds7 = CLUE({clue_file}, task, usage, 0, ShuffleMode::kGlobal, 0, -1);
EXPECT_EQ(ds7, nullptr);
// Manually terminate the pipeline
iter->Stop();
}
tests/ut/cpp/dataset/c_api_dataset_coco_test.cc
浏览文件 @ deb11464
......@@ -13,49 +13,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fstream>
#include <iostream>
#include <memory>
#include <vector>
#include <string>
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "securec.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/status.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/iterator.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/include/samplers.h"
using namespace mindspore::dataset::api;
using mindspore::MsLogLevel::ERROR;
using mindspore::ExceptionType::NoExceptionType;
using mindspore::LogStream;
using mindspore::dataset::Tensor;
using mindspore::dataset::TensorShape;
using mindspore::dataset::TensorImpl;
using mindspore::dataset::DataType;
using mindspore::dataset::Status;
using mindspore::dataset::BorderType;
using mindspore::dataset::dsize_t;
class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
TEST_F(MindDataTestPipeline, TestCocoDetection) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoDetection.";
TEST_F(MindDataTestPipeline, TestCocoDefault) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoDefault.";
// Create a Coco Dataset
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json";
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Detection", false, SequentialSampler(0, 6));
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -67,28 +43,14 @@ TEST_F(MindDataTestPipeline, TestCocoDetection) {
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
std::string expect_file[] = {"000000391895", "000000318219", "000000554625", "000000574769", "000000060623",
"000000309022"};
std::vector<std::vector<float>> expect_bbox_vector = {{10.0, 10.0, 10.0, 10.0, 70.0, 70.0, 70.0, 70.0},
{20.0, 20.0, 20.0, 20.0, 80.0, 80.0, 80.0, 80.0},
{30.0, 30.0, 30.0, 30.0}, {40.0, 40.0, 40.0, 40.0},
{50.0, 50.0, 50.0, 50.0}, {60.0, 60.0, 60.0, 60.0}};
std::vector<std::vector<uint32_t>> expect_catagoryid_list = {{1, 7}, {2, 8}, {3}, {4}, {5}, {6}};
uint64_t i = 0;
while (row.size() != 0) {
auto image = row["image"];
auto bbox = row["bbox"];
auto category_id = row["category_id"];
std::shared_ptr<Tensor> expect_image;
Tensor::CreateFromFile(folder_path + "/" + expect_file[i] + ".jpg", &expect_image);
EXPECT_EQ(*image, *expect_image);
std::shared_ptr<Tensor> expect_bbox;
dsize_t bbox_num = static_cast<dsize_t>(expect_bbox_vector[i].size() / 4);
Tensor::CreateFromVector(expect_bbox_vector[i], TensorShape({bbox_num, 4}), &expect_bbox);
EXPECT_EQ(*bbox, *expect_bbox);
std::shared_ptr<Tensor> expect_categoryid;
Tensor::CreateFromVector(expect_catagoryid_list[i], TensorShape({bbox_num, 1}), &expect_categoryid);
EXPECT_EQ(*category_id, *expect_categoryid);
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
MS_LOG(INFO) << "Tensor bbox shape: " << bbox->shape();
MS_LOG(INFO) << "Tensor category_id shape: " << category_id->shape();
iter->GetNextRow(&row);
i++;
}
......@@ -99,13 +61,13 @@ TEST_F(MindDataTestPipeline, TestCocoDetection) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCocoStuff) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoStuff.";
TEST_F(MindDataTestPipeline, TestCocoDetection) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoDetection.";
// Create a Coco Dataset
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json";
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Stuff", false, SequentialSampler(0, 6));
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Detection", false, SequentialSampler(0, 6));
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -119,27 +81,26 @@ TEST_F(MindDataTestPipeline, TestCocoStuff) {
std::string expect_file[] = {"000000391895", "000000318219", "000000554625", "000000574769", "000000060623",
"000000309022"};
std::vector<std::vector<float>> expect_segmentation_vector =
{{10.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0,
70.0, 72.0, 73.0, 74.0, 75.0, -1.0, -1.0, -1.0, -1.0, -1.0},
{20.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0,
10.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, -1.0},
{40.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 40.0, 41.0, 42.0},
{50.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, 58.0, 59.0, 60.0, 61.0, 62.0, 63.0},
{60.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0},
{60.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0}};
std::vector<std::vector<dsize_t>> expect_size = {{2, 10}, {2, 11}, {1, 12}, {1, 13}, {1, 14}, {2, 7}};
std::vector<std::vector<float>> expect_bbox_vector = {{10.0, 10.0, 10.0, 10.0, 70.0, 70.0, 70.0, 70.0},
{20.0, 20.0, 20.0, 20.0, 80.0, 80.0, 80.0, 80.0},
{30.0, 30.0, 30.0, 30.0}, {40.0, 40.0, 40.0, 40.0},
{50.0, 50.0, 50.0, 50.0}, {60.0, 60.0, 60.0, 60.0}};
std::vector<std::vector<uint32_t>> expect_catagoryid_list = {{1, 7}, {2, 8}, {3}, {4}, {5}, {6}};
uint64_t i = 0;
while (row.size() != 0) {
auto image = row["image"];
auto segmentation = row["segmentation"];
auto iscrowd = row["iscrowd"];
auto bbox = row["bbox"];
auto category_id = row["category_id"];
std::shared_ptr<Tensor> expect_image;
Tensor::CreateFromFile(folder_path + "/" + expect_file[i] + ".jpg", &expect_image);
EXPECT_EQ(*image, *expect_image);
std::shared_ptr<Tensor> expect_segmentation;
Tensor::CreateFromVector(expect_segmentation_vector[i], TensorShape(expect_size[i]), &expect_segmentation);
EXPECT_EQ(*segmentation, *expect_segmentation);
std::shared_ptr<Tensor> expect_bbox;
dsize_t bbox_num = static_cast<dsize_t>(expect_bbox_vector[i].size() / 4);
Tensor::CreateFromVector(expect_bbox_vector[i], TensorShape({bbox_num, 4}), &expect_bbox);
EXPECT_EQ(*bbox, *expect_bbox);
std::shared_ptr<Tensor> expect_categoryid;
Tensor::CreateFromVector(expect_catagoryid_list[i], TensorShape({bbox_num, 1}), &expect_categoryid);
EXPECT_EQ(*category_id, *expect_categoryid);
iter->GetNextRow(&row);
i++;
}
......@@ -150,6 +111,24 @@ TEST_F(MindDataTestPipeline, TestCocoStuff) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCocoException) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoException.";
// Create a Coco Dataset
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json";
std::string invalid_folder_path = "./NotExist";
std::string invalid_annotation_file = "./NotExistFile";
std::shared_ptr<Dataset> ds = Coco(invalid_folder_path, annotation_file);
EXPECT_EQ(ds, nullptr);
std::shared_ptr<Dataset> ds1 = Coco(folder_path, invalid_annotation_file);
EXPECT_EQ(ds1, nullptr);
std::shared_ptr<Dataset> ds2 = Coco(folder_path, annotation_file, "valid_mode");
EXPECT_EQ(ds2, nullptr);
}
TEST_F(MindDataTestPipeline, TestCocoKeypoint) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoKeypoint.";
// Create a Coco Dataset
......@@ -261,13 +240,13 @@ TEST_F(MindDataTestPipeline, TestCocoPanoptic) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCocoDefault) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoDetection.";
TEST_F(MindDataTestPipeline, TestCocoStuff) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoStuff.";
// Create a Coco Dataset
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json";
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file);
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Stuff", false, SequentialSampler(0, 6));
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -279,14 +258,29 @@ TEST_F(MindDataTestPipeline, TestCocoDefault) {
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
std::string expect_file[] = {"000000391895", "000000318219", "000000554625", "000000574769", "000000060623",
"000000309022"};
std::vector<std::vector<float>> expect_segmentation_vector =
{{10.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0,
70.0, 72.0, 73.0, 74.0, 75.0, -1.0, -1.0, -1.0, -1.0, -1.0},
{20.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0,
10.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, -1.0},
{40.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 40.0, 41.0, 42.0},
{50.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, 58.0, 59.0, 60.0, 61.0, 62.0, 63.0},
{60.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0},
{60.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0}};
std::vector<std::vector<dsize_t>> expect_size = {{2, 10}, {2, 11}, {1, 12}, {1, 13}, {1, 14}, {2, 7}};
uint64_t i = 0;
while (row.size() != 0) {
auto image = row["image"];
auto bbox = row["bbox"];
auto category_id = row["category_id"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
MS_LOG(INFO) << "Tensor bbox shape: " << bbox->shape();
MS_LOG(INFO) << "Tensor category_id shape: " << category_id->shape();
auto segmentation = row["segmentation"];
auto iscrowd = row["iscrowd"];
std::shared_ptr<Tensor> expect_image;
Tensor::CreateFromFile(folder_path + "/" + expect_file[i] + ".jpg", &expect_image);
EXPECT_EQ(*image, *expect_image);
std::shared_ptr<Tensor> expect_segmentation;
Tensor::CreateFromVector(expect_segmentation_vector[i], TensorShape(expect_size[i]), &expect_segmentation);
EXPECT_EQ(*segmentation, *expect_segmentation);
iter->GetNextRow(&row);
i++;
}
......@@ -296,21 +290,3 @@ TEST_F(MindDataTestPipeline, TestCocoDefault) {
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCocoException) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCocoDetection.";
// Create a Coco Dataset
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json";
std::string invalid_folder_path = "./NotExist";
std::string invalid_annotation_file = "./NotExistFile";
std::shared_ptr<Dataset> ds = Coco(invalid_folder_path, annotation_file);
EXPECT_EQ(ds, nullptr);
std::shared_ptr<Dataset> ds1 = Coco(folder_path, invalid_annotation_file);
EXPECT_EQ(ds1, nullptr);
std::shared_ptr<Dataset> ds2 = Coco(folder_path, annotation_file, "valid_mode");
EXPECT_EQ(ds2, nullptr);
}
tests/ut/cpp/dataset/c_api_dataset_filetext_test.cc
浏览文件 @ deb11464
......@@ -13,41 +13,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fstream>
#include <iostream>
#include <memory>
#include <vector>
#include <string>
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "./securec.h"
#include "minddata/dataset/core/client.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/global_context.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/iterator.h"
#include "minddata/dataset/include/samplers.h"
#include "minddata/dataset/include/status.h"
#include "minddata/dataset/include/transforms.h"
using namespace mindspore::dataset;
using namespace mindspore::dataset::api;
using mindspore::LogStream;
using mindspore::dataset::DataType;
using mindspore::dataset::ShuffleMode;
using mindspore::dataset::Status;
using mindspore::dataset::Tensor;
using mindspore::dataset::TensorImpl;
using mindspore::dataset::TensorShape;
using mindspore::ExceptionType::NoExceptionType;
using mindspore::MsLogLevel::ERROR;
class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
......@@ -109,6 +83,87 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetBasic) {
GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail1.";
// Attempt to create a TextFile Dataset
// with invalid samplers=-1
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1}, -1);
// Expect failure: Number of samples cannot be negative
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail2.";
// Attempt to create a TextFile Dataset
// with wrongful empty dataset_files input
std::shared_ptr<Dataset> ds = TextFile({});
// Expect failure: dataset_files is not specified
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail3) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail3.";
// Attempt to create a TextFile Dataset
// with non-existent dataset_files input
std::shared_ptr<Dataset> ds = TextFile({"notexist.txt"}, 0, ShuffleMode::kFalse);
// Expect failure: specified dataset_files does not exist
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail4) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail4.";
// Attempt to create a TextFile Dataset
// with empty string dataset_files input
std::shared_ptr<Dataset> ds = TextFile({""}, 0, ShuffleMode::kFiles);
// Expect failure: specified dataset_files does not exist
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail5) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail5.";
// Attempt to create a TextFile Dataset
// with invalid num_shards=0 value
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1}, 1, ShuffleMode::kFalse, 0);
// Expect failure: Number of shards cannot be <=0
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail6) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail6.";
// Attempt to create a TextFile Dataset
// with invalid shard_id=-1 value
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1}, 0, ShuffleMode::kFiles, -1);
// Expect failure: shard_id cannot be negative
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail7) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail7.";
// Attempt to create a TextFile Dataset
// with invalid shard_id=2 and num_shards=2 combination
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1}, 0, ShuffleMode::kGlobal, 2, 2);
// Expect failure: Cannot have shard_id >= num_shards
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleFalse1.";
// Test TextFile Dataset with two text files and no shuffle, num_parallel_workers=1
......@@ -224,22 +279,25 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse4Shard) {
GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleGlobal1A.";
// Test TextFile Dataset with 1 text file, global shuffle, num_parallel_workers=1
TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleFiles1.";
// Test TextFile Dataset with files shuffle, num_parallel_workers=1
// Set configuration
uint32_t original_seed = GlobalContext::config_manager()->seed();
uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers();
MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers;
GlobalContext::config_manager()->set_seed(246);
GlobalContext::config_manager()->set_seed(135);
GlobalContext::config_manager()->set_num_parallel_workers(1);
// Create a TextFile Dataset, with two text files
// Note: 1.txt has 3 rows
// Set shuffle to global shuffle
// Note: 2.txt has 2 rows
// Use default of all samples
// Set shuffle to files shuffle
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1}, 0, ShuffleMode::kGlobal);
std::string tf_file2 = datasets_root_path_ + "/testTextFileDataset/2.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kFiles);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset.
......@@ -252,7 +310,9 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) {
iter->GetNextRow(&row);
EXPECT_NE(row.find("text"), row.end());
std::vector<std::string> expected_result = {"Good luck to everyone.", "This is a text file.", "Be happy every day."};
std::vector<std::string> expected_result = {
"This is a text file.", "Be happy every day.", "Good luck to everyone.", "Another file.", "End of file.",
};
uint64_t i = 0;
while (row.size() != 0) {
......@@ -268,8 +328,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) {
iter->GetNextRow(&row);
}
// Expect 3 samples
EXPECT_EQ(i, 3);
// Expect 2 + 3 = 5 samples
EXPECT_EQ(i, 5);
// Manually terminate the pipeline
iter->Stop();
......@@ -279,24 +339,25 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) {
GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleGlobal1B.";
// Test TextFile Dataset with 2 text files, global shuffle, num_parallel_workers=1
TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleFiles4.";
// Test TextFile Dataset with files shuffle, num_parallel_workers=4
// Set configuration
uint32_t original_seed = GlobalContext::config_manager()->seed();
uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers();
MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers;
GlobalContext::config_manager()->set_seed(246);
GlobalContext::config_manager()->set_num_parallel_workers(1);
GlobalContext::config_manager()->set_seed(135);
GlobalContext::config_manager()->set_num_parallel_workers(4);
// Create a TextFile Dataset, with two text files
// Note: 1.txt has 3 rows
// Note: 2.txt has 2 rows
// Set shuffle to global shuffle
// Use default of all samples
// Set shuffle to files shuffle
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::string tf_file2 = datasets_root_path_ + "/testTextFileDataset/2.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kGlobal);
std::shared_ptr<Dataset> ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kFiles);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset.
......@@ -309,8 +370,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) {
iter->GetNextRow(&row);
EXPECT_NE(row.find("text"), row.end());
std::vector<std::string> expected_result = {"Another file.", "Good luck to everyone.", "This is a text file.",
"End of file.", "Be happy every day."};
std::vector<std::string> expected_result = {"This is a text file.", "Another file.", "Be happy every day.",
"End of file.", "Good luck to everyone."};
uint64_t i = 0;
while (row.size() != 0) {
......@@ -337,24 +398,22 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) {
GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleGlobal4.";
// Test TextFile Dataset with 2 text files, global shuffle, num_parallel_workers=4
TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleGlobal1A.";
// Test TextFile Dataset with 1 text file, global shuffle, num_parallel_workers=1
// Set configuration
uint32_t original_seed = GlobalContext::config_manager()->seed();
uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers();
MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers;
GlobalContext::config_manager()->set_seed(246);
GlobalContext::config_manager()->set_num_parallel_workers(4);
GlobalContext::config_manager()->set_num_parallel_workers(1);
// Create a TextFile Dataset, with two text files
// Note: 1.txt has 3 rows
// Note: 2.txt has 2 rows
// Set shuffle to global shuffle
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::string tf_file2 = datasets_root_path_ + "/testTextFileDataset/2.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kGlobal);
std::shared_ptr<Dataset> ds = TextFile({tf_file1}, 0, ShuffleMode::kGlobal);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset.
......@@ -367,8 +426,7 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) {
iter->GetNextRow(&row);
EXPECT_NE(row.find("text"), row.end());
std::vector<std::string> expected_result = {"Another file.", "Good luck to everyone.", "End of file.",
"This is a text file.", "Be happy every day."};
std::vector<std::string> expected_result = {"Good luck to everyone.", "This is a text file.", "Be happy every day."};
uint64_t i = 0;
while (row.size() != 0) {
......@@ -384,8 +442,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) {
iter->GetNextRow(&row);
}
// Expect 2 + 3 = 5 samples
EXPECT_EQ(i, 5);
// Expect 3 samples
EXPECT_EQ(i, 3);
// Manually terminate the pipeline
iter->Stop();
......@@ -395,25 +453,24 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) {
GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleFiles1.";
// Test TextFile Dataset with files shuffle, num_parallel_workers=1
TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleGlobal1B.";
// Test TextFile Dataset with 2 text files, global shuffle, num_parallel_workers=1
// Set configuration
uint32_t original_seed = GlobalContext::config_manager()->seed();
uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers();
MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers;
GlobalContext::config_manager()->set_seed(135);
GlobalContext::config_manager()->set_seed(246);
GlobalContext::config_manager()->set_num_parallel_workers(1);
// Create a TextFile Dataset, with two text files
// Note: 1.txt has 3 rows
// Note: 2.txt has 2 rows
// Use default of all samples
// Set shuffle to files shuffle
// Set shuffle to global shuffle
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::string tf_file2 = datasets_root_path_ + "/testTextFileDataset/2.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kFiles);
std::shared_ptr<Dataset> ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kGlobal);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset.
......@@ -426,9 +483,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1) {
iter->GetNextRow(&row);
EXPECT_NE(row.find("text"), row.end());
std::vector<std::string> expected_result = {
"This is a text file.", "Be happy every day.", "Good luck to everyone.", "Another file.", "End of file.",
};
std::vector<std::string> expected_result = {"Another file.", "Good luck to everyone.", "This is a text file.",
"End of file.", "Be happy every day."};
uint64_t i = 0;
while (row.size() != 0) {
......@@ -455,25 +511,24 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1) {
GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleFiles4.";
// Test TextFile Dataset with files shuffle, num_parallel_workers=4
TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetShuffleGlobal4.";
// Test TextFile Dataset with 2 text files, global shuffle, num_parallel_workers=4
// Set configuration
uint32_t original_seed = GlobalContext::config_manager()->seed();
uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers();
MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers;
GlobalContext::config_manager()->set_seed(135);
GlobalContext::config_manager()->set_seed(246);
GlobalContext::config_manager()->set_num_parallel_workers(4);
// Create a TextFile Dataset, with two text files
// Note: 1.txt has 3 rows
// Note: 2.txt has 2 rows
// Use default of all samples
// Set shuffle to files shuffle
// Set shuffle to global shuffle
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::string tf_file2 = datasets_root_path_ + "/testTextFileDataset/2.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kFiles);
std::shared_ptr<Dataset> ds = TextFile({tf_file1, tf_file2}, 0, ShuffleMode::kGlobal);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset.
......@@ -486,8 +541,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) {
iter->GetNextRow(&row);
EXPECT_NE(row.find("text"), row.end());
std::vector<std::string> expected_result = {"This is a text file.", "Another file.", "Be happy every day.",
"End of file.", "Good luck to everyone."};
std::vector<std::string> expected_result = {"Another file.", "Good luck to everyone.", "End of file.",
"This is a text file.", "Be happy every day."};
uint64_t i = 0;
while (row.size() != 0) {
......@@ -513,84 +568,3 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) {
GlobalContext::config_manager()->set_seed(original_seed);
GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail1.";
// Attempt to create a TextFile Dataset
// with invalid samplers=-1
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1}, -1);
// Expect failure: Number of samples cannot be negative
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail2.";
// Attempt to create a TextFile Dataset
// with wrongful empty dataset_files input
std::shared_ptr<Dataset> ds = TextFile({});
// Expect failure: dataset_files is not specified
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail3) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail3.";
// Attempt to create a TextFile Dataset
// with non-existent dataset_files input
std::shared_ptr<Dataset> ds = TextFile({"notexist.txt"}, 0, ShuffleMode::kFalse);
// Expect failure: specified dataset_files does not exist
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail4) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail4.";
// Attempt to create a TextFile Dataset
// with empty string dataset_files input
std::shared_ptr<Dataset> ds = TextFile({""}, 0, ShuffleMode::kFiles);
// Expect failure: specified dataset_files does not exist
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail5) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail5.";
// Attempt to create a TextFile Dataset
// with invalid num_shards=0 value
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1}, 1, ShuffleMode::kFalse, 0);
// Expect failure: Number of shards cannot be <=0
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail6) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail6.";
// Attempt to create a TextFile Dataset
// with invalid shard_id=-1 value
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1}, 0, ShuffleMode::kFiles, -1);
// Expect failure: shard_id cannot be negative
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextFileDatasetFail7) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextFileDatasetFail7.";
// Attempt to create a TextFile Dataset
// with invalid shard_id=2 and num_shards=2 combination
std::string tf_file1 = datasets_root_path_ + "/testTextFileDataset/1.txt";
std::shared_ptr<Dataset> ds = TextFile({tf_file1}, 0, ShuffleMode::kGlobal, 2, 2);
// Expect failure: Cannot have shard_id >= num_shards
EXPECT_EQ(ds, nullptr);
}
tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc
浏览文件 @ deb11464
......@@ -13,139 +13,107 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fstream>
#include <iostream>
#include <memory>
#include <vector>
#include <string>
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "securec.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/status.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/iterator.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/include/samplers.h"
using namespace mindspore::dataset::api;
using mindspore::MsLogLevel::ERROR;
using mindspore::ExceptionType::NoExceptionType;
using mindspore::LogStream;
using mindspore::dataset::Tensor;
using mindspore::dataset::TensorShape;
using mindspore::dataset::TensorImpl;
using mindspore::dataset::DataType;
using mindspore::dataset::Status;
using mindspore::dataset::BorderType;
using mindspore::dataset::dsize_t;
class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
TEST_F(MindDataTestPipeline, TestIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn.";
// Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, RandomSampler(false, 4));
TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorEmptyColumn.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, RandomSampler(false, 5));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 2;
ds = ds->Batch(batch_size);
// Create a Rename operation on ds
ds = ds->Rename({"image", "label"}, {"col1", "col2"});
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
// No columns are specified, use all columns
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
TensorShape expect({2, 28, 28, 1});
TensorShape expect0({32, 32, 3});
TensorShape expect1({});
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v->shape();
EXPECT_EQ(expect, v->shape());
}
MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape();
EXPECT_EQ(expect0, row[0]->shape());
EXPECT_EQ(expect1, row[1]->shape());
iter->GetNextRow(&row);
i++;
}
EXPECT_EQ(i, 2);
EXPECT_EQ(i, 5);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorTwoColumns.";
// Create a VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, SequentialSampler(0, 4));
TEST_F(MindDataTestPipeline, TestIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn.";
// Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, RandomSampler(false, 4));
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
// Create a Batch operation on ds
int32_t batch_size = 2;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" and "bbox" column
std::vector<std::string> columns = {"image", "bbox"};
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
std::vector<TensorShape> expect = {TensorShape({173673}), TensorShape({1, 4}),
TensorShape({173673}), TensorShape({1, 4}),
TensorShape({147025}), TensorShape({1, 4}),
TensorShape({211653}), TensorShape({1, 4})};
TensorShape expect({2, 28, 28, 1});
uint64_t i = 0;
uint64_t j = 0;
while (row.size() != 0) {
MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape();
EXPECT_EQ(2, row.size());
EXPECT_EQ(expect[j++], row[0]->shape());
EXPECT_EQ(expect[j++], row[1]->shape());
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v->shape();
EXPECT_EQ(expect, v->shape());
}
iter->GetNextRow(&row);
i++;
j = (j == expect.size()) ? 0 : j;
}
EXPECT_EQ(i, 8);
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorEmptyColumn.";
TEST_F(MindDataTestPipeline, TestIteratorReOrder) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorReOrder.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, RandomSampler(false, 5));
std::shared_ptr<Dataset> ds = Cifar10(folder_path, SequentialSampler(false, 4));
EXPECT_NE(ds, nullptr);
// Create a Rename operation on ds
ds = ds->Rename({"image", "label"}, {"col1", "col2"});
// Create a Take operation on ds
ds = ds->Take(2);
EXPECT_NE(ds, nullptr);
// No columns are specified, use all columns
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Create an iterator over the result of the above dataset
// Reorder "image" and "label" column
std::vector<std::string> columns = {"label", "image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
......@@ -154,56 +122,62 @@ TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) {
TensorShape expect0({32, 32, 3});
TensorShape expect1({});
// Check if we will catch "label" before "image" in row
std::vector<std::string> expect = {"label", "image"};
uint64_t i = 0;
while (row.size() != 0) {
MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape();
EXPECT_EQ(expect0, row[0]->shape());
EXPECT_EQ(expect1, row[1]->shape());
EXPECT_EQ(expect1, row[0]->shape());
EXPECT_EQ(expect0, row[1]->shape());
iter->GetNextRow(&row);
i++;
}
EXPECT_EQ(i, 5);
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestIteratorReOrder) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorReOrder.";
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, SequentialSampler(false, 4));
TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorTwoColumns.";
// Create a VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, SequentialSampler(0, 4));
EXPECT_NE(ds, nullptr);
// Create a Take operation on ds
ds = ds->Take(2);
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Reorder "image" and "label" column
std::vector<std::string> columns = {"label", "image"};
// Only select "image" and "bbox" column
std::vector<std::string> columns = {"image", "bbox"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
TensorShape expect0({32, 32, 3});
TensorShape expect1({});
std::vector<TensorShape> expect = {TensorShape({173673}), TensorShape({1, 4}),
TensorShape({173673}), TensorShape({1, 4}),
TensorShape({147025}), TensorShape({1, 4}),
TensorShape({211653}), TensorShape({1, 4})};
// Check if we will catch "label" before "image" in row
std::vector<std::string> expect = {"label", "image"};
uint64_t i = 0;
uint64_t j = 0;
while (row.size() != 0) {
MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape();
EXPECT_EQ(expect1, row[0]->shape());
EXPECT_EQ(expect0, row[1]->shape());
EXPECT_EQ(2, row.size());
EXPECT_EQ(expect[j++], row[0]->shape());
EXPECT_EQ(expect[j++], row[1]->shape());
iter->GetNextRow(&row);
i++;
j = (j == expect.size()) ? 0 : j;
}
EXPECT_EQ(i, 2);
EXPECT_EQ(i, 8);
// Manually terminate the pipeline
iter->Stop();
......
tests/ut/cpp/dataset/c_api_dataset_ops_test.cc
浏览文件 @ deb11464
......@@ -13,37 +13,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fstream>
#include <iostream>
#include <memory>
#include <vector>
#include <string>
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "securec.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/status.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/iterator.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/include/samplers.h"
using namespace mindspore::dataset::api;
using mindspore::MsLogLevel::ERROR;
using mindspore::ExceptionType::NoExceptionType;
using mindspore::LogStream;
using mindspore::dataset::Tensor;
using mindspore::dataset::TensorShape;
using mindspore::dataset::TensorImpl;
using mindspore::dataset::DataType;
using mindspore::dataset::Status;
using mindspore::dataset::BorderType;
using mindspore::dataset::dsize_t;
class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
......@@ -90,28 +65,26 @@ TEST_F(MindDataTestPipeline, TestBatchAndRepeat) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTensorOpsAndMap.";
// Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, RandomSampler(false, 20));
EXPECT_NE(ds, nullptr);
TEST_F(MindDataTestPipeline, TestConcatFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatFail1.";
// This case is expected to fail because the input column names of concatenated datasets are not the same
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
// Create an ImageFolder Dataset
// Column names: {"image", "label"}
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> resize_op = vision::Resize({30, 30});
EXPECT_NE(resize_op, nullptr);
std::shared_ptr<TensorOperation> center_crop_op = vision::CenterCrop({16, 16});
EXPECT_NE(center_crop_op, nullptr);
// Create a Rename operation on ds
ds2 = ds2->Rename({"image", "label"}, {"col1", "col2"});
EXPECT_NE(ds, nullptr);
// Create a Map operation on ds
ds = ds->Map({resize_op, center_crop_op});
// Create a Project operation on the ds
// Name of datasets to concat doesn't not match
ds = ds->Concat({ds2});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
......@@ -122,41 +95,51 @@ TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) {
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
EXPECT_EQ(iter, nullptr);
}
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
TEST_F(MindDataTestPipeline, TestConcatFail2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatFail2.";
// This case is expected to fail because the input dataset is empty.
EXPECT_EQ(i, 40);
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Manually terminate the pipeline
iter->Stop();
// Create a Project operation on the ds
// Input dataset to concat is empty
ds = ds->Concat({});
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestImageFolderBatchAndRepeat.";
TEST_F(MindDataTestPipeline, TestConcatSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatSuccess.";
// Create an ImageFolder Dataset
// Column names: {"image", "label"}
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
// Create a Cifar10 Dataset
// Column names: {"image", "label"}
folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, RandomSampler(false, 9));
EXPECT_NE(ds2, nullptr);
// Create a Project operation on ds
ds = ds->Project({"image"});
EXPECT_NE(ds, nullptr);
ds2 = ds2->Project({"image"});
EXPECT_NE(ds, nullptr);
// Create a Concat operation on the ds
ds = ds->Concat({ds2});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 2;
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
......@@ -168,7 +151,6 @@ TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) {
// Iterate the dataset and get each row
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
uint64_t i = 0;
while (row.size() != 0) {
i++;
......@@ -177,32 +159,39 @@ TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) {
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 10);
EXPECT_EQ(i, 19);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestShuffleDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestShuffleDataset.";
TEST_F(MindDataTestPipeline, TestConcatSuccess2) {
// Test "+" operator to concat two datasets
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatSuccess2.";
// Create an ImageFolder Dataset
// Column names: {"image", "label"}
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Shuffle operation on ds
int32_t shuffle_size = 10;
ds = ds->Shuffle(shuffle_size);
// Create a Cifar10 Dataset
// Column names: {"image", "label"}
folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, RandomSampler(false, 9));
EXPECT_NE(ds2, nullptr);
// Create a Project operation on ds
ds = ds->Project({"image"});
EXPECT_NE(ds, nullptr);
ds2 = ds2->Project({"image"});
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
// Create a Concat operation on the ds
ds = ds + ds2;
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 2;
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
......@@ -214,7 +203,6 @@ TEST_F(MindDataTestPipeline, TestShuffleDataset) {
// Iterate the dataset and get each row
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
uint64_t i = 0;
while (row.size() != 0) {
i++;
......@@ -223,23 +211,27 @@ TEST_F(MindDataTestPipeline, TestShuffleDataset) {
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 10);
EXPECT_EQ(i, 19);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestSkipDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSkipDataset.";
TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestImageFolderBatchAndRepeat.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Skip operation on ds
int32_t count = 3;
ds = ds->Skip(count);
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 2;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -258,40 +250,42 @@ TEST_F(MindDataTestPipeline, TestSkipDataset) {
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
MS_LOG(INFO) << "Number of rows: " << i;
// Expect 10-3=7 rows
EXPECT_EQ(i, 7);
EXPECT_EQ(i, 10);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestSkipDatasetError1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSkipDatasetError1.";
TEST_F(MindDataTestPipeline, TestProjectMap) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestProjectMap.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Skip operation on ds with invalid count input
int32_t count = -1;
ds = ds->Skip(count);
// Expect nullptr for invalid input skip_count
EXPECT_EQ(ds, nullptr);
}
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
TEST_F(MindDataTestPipeline, TestTakeDatasetDefault) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTakeDatasetDefault.";
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> random_vertical_flip_op = vision::RandomVerticalFlip(0.5);
EXPECT_NE(random_vertical_flip_op, nullptr);
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 7));
// Create a Map operation on ds
ds = ds->Map({random_vertical_flip_op}, {}, {}, {"image", "label"});
EXPECT_NE(ds, nullptr);
// Create a Take operation on ds, dafault count = -1
ds = ds->Take();
// Create a Project operation on ds
std::vector<std::string> column_project = {"image"};
ds = ds->Project(column_project);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -310,25 +304,33 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetDefault) {
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
MS_LOG(INFO) << "Number of rows: " << i;
// Expect 7 rows
EXPECT_EQ(i, 7);
EXPECT_EQ(i, 20);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestTakeDatasetNormal) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTakeDatasetNormal.";
TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) {
MS_LOG(INFO) << "Doing MindDataTestPipeline.TestProjectMapAutoInjection";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 8));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Take operation on ds
ds = ds->Take(5);
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> resize_op = vision::Resize({30, 30});
EXPECT_NE(resize_op, nullptr);
// Create a Map operation on ds
// {"image"} is the project columns. This will trigger auto injection of ProjectOp after MapOp.
ds = ds->Map({resize_op}, {}, {}, {"image"});
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -340,39 +342,47 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetNormal) {
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
// 'label' is dropped during the project op
EXPECT_EQ(row.find("label"), row.end());
// 'image' column should still exist
EXPECT_NE(row.find("image"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
EXPECT_EQ(image->shape()[0], 30);
iter->GetNextRow(&row);
}
MS_LOG(INFO) << "Number of rows: " << i;
// Expect 5 rows
EXPECT_EQ(i, 5);
EXPECT_EQ(i, 20);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestTakeDatasetError1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTakeDatasetError1.";
TEST_F(MindDataTestPipeline, TestRenameFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRenameFail.";
// We expect this test to fail because input and output in Rename are not the same size
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Take operation on ds with invalid count input
int32_t count = -5;
ds = ds->Take(count);
// Expect nullptr for invalid input take_count
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
// Create a Rename operation on ds
ds = ds->Rename({"image", "label"}, {"col2"});
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestProjectMap) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestProjectMap.";
TEST_F(MindDataTestPipeline, TestRenameSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRenameSuccess.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
......@@ -384,17 +394,8 @@ TEST_F(MindDataTestPipeline, TestProjectMap) {
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> random_vertical_flip_op = vision::RandomVerticalFlip(0.5);
EXPECT_NE(random_vertical_flip_op, nullptr);
// Create a Map operation on ds
ds = ds->Map({random_vertical_flip_op}, {}, {}, {"image", "label"});
EXPECT_NE(ds, nullptr);
// Create a Project operation on ds
std::vector<std::string> column_project = {"image"};
ds = ds->Project(column_project);
// Create a Rename operation on ds
ds = ds->Rename({"image", "label"}, {"col1", "col2"});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
......@@ -412,9 +413,14 @@ TEST_F(MindDataTestPipeline, TestProjectMap) {
iter->GetNextRow(&row);
uint64_t i = 0;
EXPECT_NE(row.find("col1"), row.end());
EXPECT_NE(row.find("col2"), row.end());
EXPECT_EQ(row.find("image"), row.end());
EXPECT_EQ(row.find("label"), row.end());
while (row.size() != 0) {
i++;
auto image = row["image"];
auto image = row["col1"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
......@@ -425,27 +431,27 @@ TEST_F(MindDataTestPipeline, TestProjectMap) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) {
MS_LOG(INFO) << "Doing MindDataTestPipeline.TestProjectMapAutoInjection";
TEST_F(MindDataTestPipeline, TestShuffleDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestShuffleDataset.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Shuffle operation on ds
int32_t shuffle_size = 10;
ds = ds->Shuffle(shuffle_size);
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> resize_op = vision::Resize({30, 30});
EXPECT_NE(resize_op, nullptr);
// Create a Map operation on ds
// {"image"} is the project columns. This will trigger auto injection of ProjectOp after MapOp.
ds = ds->Map({resize_op}, {}, {}, {"image"});
// Create a Batch operation on ds
int32_t batch_size = 2;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -457,64 +463,31 @@ TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) {
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
// 'label' is dropped during the project op
EXPECT_EQ(row.find("label"), row.end());
// 'image' column should still exist
EXPECT_NE(row.find("image"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
EXPECT_EQ(image->shape()[0], 30);
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 20);
EXPECT_EQ(i, 10);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestZipSuccess) {
// Testing the member zip() function
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipSuccess.";
TEST_F(MindDataTestPipeline, TestSkipDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSkipDataset.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Project operation on ds
std::vector<std::string> column_project = {"image"};
ds = ds->Project(column_project);
EXPECT_NE(ds, nullptr);
// Create an ImageFolder Dataset
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds1, nullptr);
// Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names)
ds1 = ds1->Rename({"image", "label"}, {"col1", "col2"});
EXPECT_NE(ds1, nullptr);
folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, RandomSampler(false, 10));
EXPECT_NE(ds2, nullptr);
// Create a Project operation on ds
column_project = {"label"};
ds2 = ds2->Project(column_project);
EXPECT_NE(ds2, nullptr);
// Create a Zip operation on the datasets
ds = ds->Zip({ds1, ds2});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
// Create a Skip operation on ds
int32_t count = 3;
ds = ds->Skip(count);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -526,13 +499,6 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) {
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
// Check zipped column names
EXPECT_EQ(row.size(), 4);
EXPECT_NE(row.find("image"), row.end());
EXPECT_NE(row.find("label"), row.end());
EXPECT_NE(row.find("col1"), row.end());
EXPECT_NE(row.find("col2"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
i++;
......@@ -540,35 +506,40 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) {
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
MS_LOG(INFO) << "Number of rows: " << i;
EXPECT_EQ(i, 10);
// Expect 10-3=7 rows
EXPECT_EQ(i, 7);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestZipSuccess2) {
// Testing the static zip() function
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipSuccess2.";
TEST_F(MindDataTestPipeline, TestSkipDatasetError1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSkipDatasetError1.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 9));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds2, nullptr);
// Create a Rename operation on ds (so that the 2 datasets we are going to zip have distinct column names)
ds = ds->Rename({"image", "label"}, {"col1", "col2"});
EXPECT_NE(ds, nullptr);
// Create a Skip operation on ds with invalid count input
int32_t count = -1;
ds = ds->Skip(count);
// Expect nullptr for invalid input skip_count
EXPECT_EQ(ds, nullptr);
}
// Create a Zip operation on the datasets
ds = Zip({ds, ds2});
TEST_F(MindDataTestPipeline, TestTakeDatasetDefault) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTakeDatasetDefault.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 7));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
// Create a Take operation on ds, dafault count = -1
ds = ds->Take();
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -580,13 +551,6 @@ TEST_F(MindDataTestPipeline, TestZipSuccess2) {
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
// Check zipped column names
EXPECT_EQ(row.size(), 4);
EXPECT_NE(row.find("image"), row.end());
EXPECT_NE(row.find("label"), row.end());
EXPECT_NE(row.find("col1"), row.end());
EXPECT_NE(row.find("col2"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
i++;
......@@ -594,63 +558,73 @@ TEST_F(MindDataTestPipeline, TestZipSuccess2) {
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
MS_LOG(INFO) << "Number of rows: " << i;
EXPECT_EQ(i, 9);
// Expect 7 rows
EXPECT_EQ(i, 7);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestZipFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipFail.";
// We expect this test to fail because we are the both datasets we are zipping have "image" and "label" columns
// and zip doesn't accept datasets with same column names
TEST_F(MindDataTestPipeline, TestTakeDatasetError1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTakeDatasetError1.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create an ImageFolder Dataset
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds1, nullptr);
// Create a Take operation on ds with invalid count input
int32_t count = -5;
ds = ds->Take(count);
// Expect nullptr for invalid input take_count
EXPECT_EQ(ds, nullptr);
}
// Create a Zip operation on the datasets
ds = Zip({ds, ds1});
TEST_F(MindDataTestPipeline, TestTakeDatasetNormal) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTakeDatasetNormal.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 8));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
// Create a Take operation on ds
ds = ds->Take(5);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_EQ(iter, nullptr);
}
EXPECT_NE(iter, nullptr);
TEST_F(MindDataTestPipeline, TestZipFail2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipFail2.";
// This case is expected to fail because the input dataset is empty.
// Iterate the dataset and get each row
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
MS_LOG(INFO) << "Number of rows: " << i;
// Create a Zip operation on the datasets
// Input dataset to zip is empty
ds = Zip({});
EXPECT_EQ(ds, nullptr);
// Expect 5 rows
EXPECT_EQ(i, 5);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestRenameSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRenameSuccess.";
TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTensorOpsAndMap.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
// Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, RandomSampler(false, 20));
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
......@@ -658,8 +632,15 @@ TEST_F(MindDataTestPipeline, TestRenameSuccess) {
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
// Create a Rename operation on ds
ds = ds->Rename({"image", "label"}, {"col1", "col2"});
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> resize_op = vision::Resize({30, 30});
EXPECT_NE(resize_op, nullptr);
std::shared_ptr<TensorOperation> center_crop_op = vision::CenterCrop({16, 16});
EXPECT_NE(center_crop_op, nullptr);
// Create a Map operation on ds
ds = ds->Map({resize_op, center_crop_op});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
......@@ -677,66 +658,97 @@ TEST_F(MindDataTestPipeline, TestRenameSuccess) {
iter->GetNextRow(&row);
uint64_t i = 0;
EXPECT_NE(row.find("col1"), row.end());
EXPECT_NE(row.find("col2"), row.end());
EXPECT_EQ(row.find("image"), row.end());
EXPECT_EQ(row.find("label"), row.end());
while (row.size() != 0) {
i++;
auto image = row["col1"];
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 20);
EXPECT_EQ(i, 40);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestRenameFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRenameFail.";
// We expect this test to fail because input and output in Rename are not the same size
TEST_F(MindDataTestPipeline, TestZipFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipFail.";
// We expect this test to fail because we are the both datasets we are zipping have "image" and "label" columns
// and zip doesn't accept datasets with same column names
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
// Create an ImageFolder Dataset
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds1, nullptr);
// Create a Zip operation on the datasets
ds = Zip({ds, ds1});
EXPECT_NE(ds, nullptr);
// Create a Rename operation on ds
ds = ds->Rename({"image", "label"}, {"col2"});
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestZipFail2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipFail2.";
// This case is expected to fail because the input dataset is empty.
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Zip operation on the datasets
// Input dataset to zip is empty
ds = Zip({});
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestConcatSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatSuccess.";
TEST_F(MindDataTestPipeline, TestZipSuccess) {
// Testing the member zip() function
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipSuccess.";
// Create an ImageFolder Dataset
// Column names: {"image", "label"}
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Cifar10 Dataset
// Column names: {"image", "label"}
// Create a Project operation on ds
std::vector<std::string> column_project = {"image"};
ds = ds->Project(column_project);
EXPECT_NE(ds, nullptr);
// Create an ImageFolder Dataset
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds1, nullptr);
// Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names)
ds1 = ds1->Rename({"image", "label"}, {"col1", "col2"});
EXPECT_NE(ds1, nullptr);
folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, RandomSampler(false, 9));
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, RandomSampler(false, 10));
EXPECT_NE(ds2, nullptr);
// Create a Project operation on ds
ds = ds->Project({"image"});
EXPECT_NE(ds, nullptr);
ds2 = ds2->Project({"image"});
EXPECT_NE(ds, nullptr);
column_project = {"label"};
ds2 = ds2->Project(column_project);
EXPECT_NE(ds2, nullptr);
// Create a Concat operation on the ds
ds = ds->Concat({ds2});
// Create a Zip operation on the datasets
ds = ds->Zip({ds1, ds2});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
......@@ -752,6 +764,14 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess) {
// Iterate the dataset and get each row
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
// Check zipped column names
EXPECT_EQ(row.size(), 4);
EXPECT_NE(row.find("image"), row.end());
EXPECT_NE(row.find("label"), row.end());
EXPECT_NE(row.find("col1"), row.end());
EXPECT_NE(row.find("col2"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
i++;
......@@ -760,35 +780,29 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess) {
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 19);
EXPECT_EQ(i, 10);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestConcatSuccess2) {
// Test "+" operator to concat two datasets
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatSuccess2.";
TEST_F(MindDataTestPipeline, TestZipSuccess2) {
// Testing the static zip() function
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestZipSuccess2.";
// Create an ImageFolder Dataset
// Column names: {"image", "label"}
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 9));
EXPECT_NE(ds, nullptr);
// Create a Cifar10 Dataset
// Column names: {"image", "label"}
folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, RandomSampler(false, 9));
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds2, nullptr);
// Create a Project operation on ds
ds = ds->Project({"image"});
EXPECT_NE(ds, nullptr);
ds2 = ds2->Project({"image"});
// Create a Rename operation on ds (so that the 2 datasets we are going to zip have distinct column names)
ds = ds->Rename({"image", "label"}, {"col1", "col2"});
EXPECT_NE(ds, nullptr);
// Create a Concat operation on the ds
ds = ds + ds2;
// Create a Zip operation on the datasets
ds = Zip({ds, ds2});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
......@@ -804,6 +818,14 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess2) {
// Iterate the dataset and get each row
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
// Check zipped column names
EXPECT_EQ(row.size(), 4);
EXPECT_NE(row.find("image"), row.end());
EXPECT_NE(row.find("label"), row.end());
EXPECT_NE(row.find("col1"), row.end());
EXPECT_NE(row.find("col2"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
i++;
......@@ -812,54 +834,8 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess2) {
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 19);
EXPECT_EQ(i, 9);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestConcatFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatFail1.";
// This case is expected to fail because the input column names of concatenated datasets are not the same
// Create an ImageFolder Dataset
// Column names: {"image", "label"}
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Rename operation on ds
ds2 = ds2->Rename({"image", "label"}, {"col1", "col2"});
EXPECT_NE(ds, nullptr);
// Create a Project operation on the ds
// Name of datasets to concat doesn't not match
ds = ds->Concat({ds2});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestConcatFail2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConcatFail2.";
// This case is expected to fail because the input dataset is empty.
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Project operation on the ds
// Input dataset to concat is empty
ds = ds->Concat({});
EXPECT_EQ(ds, nullptr);
}
tests/ut/cpp/dataset/c_api_dataset_voc_test.cc
浏览文件 @ deb11464
......@@ -13,54 +13,30 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fstream>
#include <iostream>
#include <memory>
#include <vector>
#include <string>
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "securec.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/status.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/iterator.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/include/samplers.h"
#include "minddata/dataset/engine/datasetops/source/voc_op.h"
#include "minddata/dataset/include/datasets.h"
using namespace mindspore::dataset::api;
using mindspore::MsLogLevel::ERROR;
using mindspore::ExceptionType::NoExceptionType;
using mindspore::LogStream;
using mindspore::dataset::Tensor;
using mindspore::dataset::TensorShape;
using mindspore::dataset::TensorImpl;
using mindspore::dataset::DataType;
using mindspore::dataset::Status;
using mindspore::dataset::BorderType;
using mindspore::dataset::dsize_t;
class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
TEST_F(MindDataTestPipeline, TestVOCSegmentation) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCSegmentation.";
TEST_F(MindDataTestPipeline, TestVOCClassIndex) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCClassIndex.";
// Create a VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", {}, false, SequentialSampler(0, 3));
EXPECT_NE(ds, nullptr);
std::map<std::string, int32_t> class_index;
class_index["car"] = 0;
class_index["cat"] = 1;
class_index["train"] = 9;
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6));
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -72,23 +48,20 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) {
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
// Check if VOCOp read correct images/targets
using Tensor = mindspore::dataset::Tensor;
std::string expect_file[] = {"32", "33", "39", "32", "33", "39"};
// Check if VOCOp read correct labels
// When we provide class_index, label of ["car","cat","train"] become [0,1,9]
std::shared_ptr<Tensor> expect_label;
Tensor::CreateFromMemory(TensorShape({1, 1}), DataType(DataType::DE_UINT32), nullptr, &expect_label);
uint32_t expect[] = {9, 9, 9, 1, 1, 0};
uint64_t i = 0;
while (row.size() != 0) {
auto image = row["image"];
auto target = row["target"];
auto label = row["label"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
MS_LOG(INFO) << "Tensor target shape: " << target->shape();
std::shared_ptr<Tensor> expect_image;
Tensor::CreateFromFile(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg", &expect_image);
EXPECT_EQ(*image, *expect_image);
std::shared_ptr<Tensor> expect_target;
Tensor::CreateFromFile(folder_path + "/SegmentationClass/" + expect_file[i] + ".png", &expect_target);
EXPECT_EQ(*target, *expect_target);
MS_LOG(INFO) << "Tensor label shape: " << label->shape();
expect_label->SetItemAt({0, 0}, expect[i]);
EXPECT_EQ(*label, *expect_label);
iter->GetNextRow(&row);
i++;
......@@ -100,33 +73,6 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestVOCSegmentationError1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCSegmentationError1.";
// Create a VOC Dataset
std::map<std::string, int32_t> class_index;
class_index["car"] = 0;
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", class_index, false, RandomSampler(false, 6));
// Expect nullptr for segmentation task with class_index
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestVOCInvalidTaskOrMode) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCInvalidTaskOrMode.";
// Create a VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds_1 = VOC(folder_path, "Classification", "train", {}, false, SequentialSampler(0, 3));
// Expect nullptr for invalid task
EXPECT_EQ(ds_1, nullptr);
std::shared_ptr<Dataset> ds_2 = VOC(folder_path, "Segmentation", "validation", {}, false, RandomSampler(false, 4));
// Expect nullptr for invalid mode
EXPECT_EQ(ds_2, nullptr);
}
TEST_F(MindDataTestPipeline, TestVOCDetection) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCDetection.";
......@@ -173,17 +119,31 @@ TEST_F(MindDataTestPipeline, TestVOCDetection) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestVOCClassIndex) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCClassIndex.";
TEST_F(MindDataTestPipeline, TestVOCInvalidTaskOrMode) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCInvalidTaskOrMode.";
// Create a VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::map<std::string, int32_t> class_index;
class_index["car"] = 0;
class_index["cat"] = 1;
class_index["train"] = 9;
std::shared_ptr<Dataset> ds_1 = VOC(folder_path, "Classification", "train", {}, false, SequentialSampler(0, 3));
// Expect nullptr for invalid task
EXPECT_EQ(ds_1, nullptr);
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6));
std::shared_ptr<Dataset> ds_2 = VOC(folder_path, "Segmentation", "validation", {}, false, RandomSampler(false, 4));
// Expect nullptr for invalid mode
EXPECT_EQ(ds_2, nullptr);
}
TEST_F(MindDataTestPipeline, TestVOCSegmentation) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCSegmentation.";
// Create a VOC Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", {}, false, SequentialSampler(0, 3));
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -195,20 +155,23 @@ TEST_F(MindDataTestPipeline, TestVOCClassIndex) {
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
// Check if VOCOp read correct labels
// When we provide class_index, label of ["car","cat","train"] become [0,1,9]
std::shared_ptr<Tensor> expect_label;
Tensor::CreateFromMemory(TensorShape({1, 1}), DataType(DataType::DE_UINT32), nullptr, &expect_label);
uint32_t expect[] = {9, 9, 9, 1, 1, 0};
// Check if VOCOp read correct images/targets
using Tensor = mindspore::dataset::Tensor;
std::string expect_file[] = {"32", "33", "39", "32", "33", "39"};
uint64_t i = 0;
while (row.size() != 0) {
auto image = row["image"];
auto label = row["label"];
auto target = row["target"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
MS_LOG(INFO) << "Tensor label shape: " << label->shape();
expect_label->SetItemAt({0, 0}, expect[i]);
EXPECT_EQ(*label, *expect_label);
MS_LOG(INFO) << "Tensor target shape: " << target->shape();
std::shared_ptr<Tensor> expect_image;
Tensor::CreateFromFile(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg", &expect_image);
EXPECT_EQ(*image, *expect_image);
std::shared_ptr<Tensor> expect_target;
Tensor::CreateFromFile(folder_path + "/SegmentationClass/" + expect_file[i] + ".png", &expect_target);
EXPECT_EQ(*target, *expect_target);
iter->GetNextRow(&row);
i++;
......@@ -219,3 +182,16 @@ TEST_F(MindDataTestPipeline, TestVOCClassIndex) {
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestVOCSegmentationError1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVOCSegmentationError1.";
// Create a VOC Dataset
std::map<std::string, int32_t> class_index;
class_index["car"] = 0;
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", class_index, false, RandomSampler(false, 6));
// Expect nullptr for segmentation task with class_index
EXPECT_EQ(ds, nullptr);
}
tests/ut/cpp/dataset/c_api_datasets_test.cc
浏览文件 @ deb11464
......@@ -13,58 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fstream>
#include <iostream>
#include <memory>
#include <vector>
#include <string>
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "securec.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/status.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/iterator.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/include/samplers.h"
using namespace mindspore::dataset::api;
using mindspore::MsLogLevel::ERROR;
using mindspore::ExceptionType::NoExceptionType;
using mindspore::LogStream;
using mindspore::dataset::Tensor;
using mindspore::dataset::TensorShape;
using mindspore::dataset::TensorImpl;
using mindspore::dataset::DataType;
using mindspore::dataset::Status;
using mindspore::dataset::BorderType;
using mindspore::dataset::dsize_t;
class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
TEST_F(MindDataTestPipeline, TestMnistFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMnistFail1.";
// Create a Mnist Dataset
std::shared_ptr<Dataset> ds = Mnist("", RandomSampler(false, 10));
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestImageFolderFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestImageFolderFail1.";
// Create an ImageFolder Dataset
std::shared_ptr<Dataset> ds = ImageFolder("", true, nullptr);
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestCelebADataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCelebADataset.";
......@@ -158,3 +117,19 @@ TEST_F(MindDataTestPipeline, TestCelebAException) {
std::shared_ptr<Dataset> ds1 = CelebA(folder_path, invalid_dataset_type);
EXPECT_EQ(ds1, nullptr);
}
TEST_F(MindDataTestPipeline, TestImageFolderFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestImageFolderFail1.";
// Create an ImageFolder Dataset
std::shared_ptr<Dataset> ds = ImageFolder("", true, nullptr);
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestMnistFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMnistFail1.";
// Create a Mnist Dataset
std::shared_ptr<Dataset> ds = Mnist("", RandomSampler(false, 10));
EXPECT_EQ(ds, nullptr);
}
tests/ut/cpp/dataset/c_api_samplers_test.cc
浏览文件 @ deb11464
......@@ -13,33 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fstream>
#include <iostream>
#include <memory>
#include <vector>
#include <string>
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "securec.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/status.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/iterator.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/include/samplers.h"
using namespace mindspore::dataset::api;
using mindspore::MsLogLevel::ERROR;
using mindspore::ExceptionType::NoExceptionType;
using mindspore::LogStream;
using mindspore::dataset::Tensor;
using mindspore::dataset::Status;
using mindspore::dataset::BorderType;
class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
......
tests/ut/cpp/dataset/c_api_transforms_test.cc
浏览文件 @ deb11464
......@@ -13,64 +13,43 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fstream>
#include <iostream>
#include <memory>
#include <vector>
#include <string>
#include "utils/log_adapter.h"
#include "utils/ms_utils.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "securec.h"
#include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/include/status.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/include/iterator.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/include/samplers.h"
using namespace mindspore::dataset::api;
using mindspore::LogStream;
using mindspore::dataset::BorderType;
using mindspore::dataset::Status;
using mindspore::dataset::Tensor;
using mindspore::ExceptionType::NoExceptionType;
using mindspore::MsLogLevel::ERROR;
class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
TEST_F(MindDataTestPipeline, TestUniformAugWithOps) {
// Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, RandomSampler(false, 20));
TEST_F(MindDataTestPipeline, TestCutOut) {
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
int32_t repeat_num = 1;
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> resize_op = vision::Resize({30, 30});
EXPECT_NE(resize_op, nullptr);
std::shared_ptr<TensorOperation> random_crop_op = vision::RandomCrop({28, 28});
EXPECT_NE(random_crop_op, nullptr);
std::shared_ptr<TensorOperation> center_crop_op = vision::CenterCrop({16, 16});
EXPECT_NE(center_crop_op, nullptr);
std::shared_ptr<TensorOperation> cut_out1 = vision::CutOut(30, 5);
EXPECT_NE(cut_out1, nullptr);
std::shared_ptr<TensorOperation> uniform_aug_op = vision::UniformAugment({random_crop_op, center_crop_op}, 2);
EXPECT_NE(uniform_aug_op, nullptr);
std::shared_ptr<TensorOperation> cut_out2 = vision::CutOut(30);
EXPECT_NE(cut_out2, nullptr);
// Create a Map operation on ds
ds = ds->Map({resize_op, uniform_aug_op});
ds = ds->Map({cut_out1, cut_out2});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -84,10 +63,10 @@ TEST_F(MindDataTestPipeline, TestUniformAugWithOps) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 20);
......@@ -96,10 +75,10 @@ TEST_F(MindDataTestPipeline, TestUniformAugWithOps) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestRandomFlip) {
TEST_F(MindDataTestPipeline, TestDecode) {
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
......@@ -108,14 +87,11 @@ TEST_F(MindDataTestPipeline, TestRandomFlip) {
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> random_vertical_flip_op = vision::RandomVerticalFlip(0.5);
EXPECT_NE(random_vertical_flip_op, nullptr);
std::shared_ptr<TensorOperation> random_horizontal_flip_op = vision::RandomHorizontalFlip(0.5);
EXPECT_NE(random_horizontal_flip_op, nullptr);
std::shared_ptr<TensorOperation> decode = vision::Decode(true);
EXPECT_NE(decode, nullptr);
// Create a Map operation on ds
ds = ds->Map({random_vertical_flip_op, random_horizontal_flip_op});
ds = ds->Map({decode});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
......@@ -139,37 +115,34 @@ TEST_F(MindDataTestPipeline, TestRandomFlip) {
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 20);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess1) {
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, RandomSampler(false, 10));
TEST_F(MindDataTestPipeline, TestHwcToChw) {
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 5;
ds = ds->Batch(batch_size);
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> one_hot_op = vision::OneHot(10);
EXPECT_NE(one_hot_op, nullptr);
std::shared_ptr<TensorOperation> channel_swap = vision::HWC2CHW();
EXPECT_NE(channel_swap, nullptr);
// Create a Map operation on ds
ds = ds->Map({one_hot_op},{"label"});
ds = ds->Map({channel_swap});
EXPECT_NE(ds, nullptr);
std::shared_ptr<TensorOperation> mixup_batch_op = vision::MixUpBatch(0.5);
EXPECT_NE(mixup_batch_op, nullptr);
// Create a Map operation on ds
ds = ds->Map({mixup_batch_op}, {"image", "label"});
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -186,16 +159,18 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess1) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
// check if the image is in NCHW
EXPECT_EQ(batch_size == image->shape()[0] && 3 == image->shape()[1]
&& 2268 == image->shape()[2] && 4032 == image->shape()[3], true);
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 2);
EXPECT_EQ(i, 20);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) {
TEST_F(MindDataTestPipeline, TestMixUpBatchFail1) {
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, RandomSampler(false, 10));
......@@ -214,7 +189,30 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) {
ds = ds->Map({one_hot_op},{"label"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<TensorOperation> mixup_batch_op = vision::MixUpBatch();
std::shared_ptr<TensorOperation> mixup_batch_op = vision::MixUpBatch(-1);
EXPECT_EQ(mixup_batch_op, nullptr);
}
TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess1) {
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 5;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> one_hot_op = vision::OneHot(10);
EXPECT_NE(one_hot_op, nullptr);
// Create a Map operation on ds
ds = ds->Map({one_hot_op},{"label"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<TensorOperation> mixup_batch_op = vision::MixUpBatch(0.5);
EXPECT_NE(mixup_batch_op, nullptr);
// Create a Map operation on ds
......@@ -244,7 +242,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestMixUpBatchFail1) {
TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) {
// Create a Cifar10 Dataset
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
std::shared_ptr<Dataset> ds = Cifar10(folder_path, RandomSampler(false, 10));
......@@ -263,38 +261,11 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchFail1) {
ds = ds->Map({one_hot_op},{"label"});
EXPECT_NE(ds, nullptr);
std::shared_ptr<TensorOperation> mixup_batch_op = vision::MixUpBatch(-1);
EXPECT_EQ(mixup_batch_op, nullptr);
}
TEST_F(MindDataTestPipeline, TestPad) {
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
int32_t repeat_num = 2;
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> pad_op1 = vision::Pad({1, 2, 3, 4}, {0}, BorderType::kSymmetric);
EXPECT_NE(pad_op1, nullptr);
std::shared_ptr<TensorOperation> pad_op2 = vision::Pad({1}, {1, 1, 1}, BorderType::kEdge);
EXPECT_NE(pad_op2, nullptr);
std::shared_ptr<TensorOperation> pad_op3 = vision::Pad({1, 4});
EXPECT_NE(pad_op3, nullptr);
std::shared_ptr<TensorOperation> mixup_batch_op = vision::MixUpBatch();
EXPECT_NE(mixup_batch_op, nullptr);
// Create a Map operation on ds
ds = ds->Map({pad_op1, pad_op2, pad_op3});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
ds = ds->Map({mixup_batch_op}, {"image", "label"});
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......@@ -314,13 +285,13 @@ TEST_F(MindDataTestPipeline, TestPad) {
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 20);
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestCutOut) {
TEST_F(MindDataTestPipeline, TestNormalize) {
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
......@@ -332,14 +303,11 @@ TEST_F(MindDataTestPipeline, TestCutOut) {
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> cut_out1 = vision::CutOut(30, 5);
EXPECT_NE(cut_out1, nullptr);
std::shared_ptr<TensorOperation> cut_out2 = vision::CutOut(30);
EXPECT_NE(cut_out2, nullptr);
std::shared_ptr<TensorOperation> normalize = vision::Normalize({121.0, 115.0, 100.0}, {70.0, 68.0, 71.0});
EXPECT_NE(normalize, nullptr);
// Create a Map operation on ds
ds = ds->Map({cut_out1, cut_out2});
ds = ds->Map({normalize});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
......@@ -370,7 +338,7 @@ TEST_F(MindDataTestPipeline, TestCutOut) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestNormalize) {
TEST_F(MindDataTestPipeline, TestPad) {
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
......@@ -382,11 +350,17 @@ TEST_F(MindDataTestPipeline, TestNormalize) {
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> normalize = vision::Normalize({121.0, 115.0, 100.0}, {70.0, 68.0, 71.0});
EXPECT_NE(normalize, nullptr);
std::shared_ptr<TensorOperation> pad_op1 = vision::Pad({1, 2, 3, 4}, {0}, BorderType::kSymmetric);
EXPECT_NE(pad_op1, nullptr);
std::shared_ptr<TensorOperation> pad_op2 = vision::Pad({1}, {1, 1, 1}, BorderType::kEdge);
EXPECT_NE(pad_op2, nullptr);
std::shared_ptr<TensorOperation> pad_op3 = vision::Pad({1, 4});
EXPECT_NE(pad_op3, nullptr);
// Create a Map operation on ds
ds = ds->Map({normalize});
ds = ds->Map({pad_op1, pad_op2, pad_op3});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
......@@ -405,10 +379,10 @@ TEST_F(MindDataTestPipeline, TestNormalize) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 20);
......@@ -417,10 +391,26 @@ TEST_F(MindDataTestPipeline, TestNormalize) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestDecode) {
TEST_F(MindDataTestPipeline, TestRandomAffineFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomAffineFail with invalid params.";
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> affine = vision::RandomAffine({0.0, 0.0}, {});
EXPECT_EQ(affine, nullptr);
// Invalid number of values for translate
affine = vision::RandomAffine({0.0, 0.0}, {1, 1, 1, 1});
EXPECT_EQ(affine, nullptr);
// Invalid number of values for shear
affine = vision::RandomAffine({30.0, 30.0}, {0.0, 0.0}, {2.0, 2.0}, {10.0, 10.0});
EXPECT_EQ(affine, nullptr);
}
TEST_F(MindDataTestPipeline, TestRandomAffineSuccess1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomAffineSuccess1 with non-default params.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 10));
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
......@@ -429,11 +419,12 @@ TEST_F(MindDataTestPipeline, TestDecode) {
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> decode = vision::Decode(true);
EXPECT_NE(decode, nullptr);
std::shared_ptr<TensorOperation> affine =
vision::RandomAffine({30.0, 30.0}, {0.0, 0.0}, {2.0, 2.0}, {10.0, 10.0, 20.0, 20.0});
EXPECT_NE(affine, nullptr);
// Create a Map operation on ds
ds = ds->Map({decode});
ds = ds->Map({affine});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
......@@ -457,13 +448,16 @@ TEST_F(MindDataTestPipeline, TestDecode) {
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 20);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestHwcToChw) {
TEST_F(MindDataTestPipeline, TestRandomAffineSuccess2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomAffineSuccess2 with default params.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
......@@ -475,11 +469,11 @@ TEST_F(MindDataTestPipeline, TestHwcToChw) {
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> channel_swap = vision::HWC2CHW();
EXPECT_NE(channel_swap, nullptr);
std::shared_ptr<TensorOperation> affine = vision::RandomAffine({0.0, 0.0});
EXPECT_NE(affine, nullptr);
// Create a Map operation on ds
ds = ds->Map({channel_swap});
ds = ds->Map({affine});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
......@@ -501,11 +495,9 @@ TEST_F(MindDataTestPipeline, TestHwcToChw) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
// check if the image is in NCHW
EXPECT_EQ(batch_size == image->shape()[0] && 3 == image->shape()[1]
&& 2268 == image->shape()[2] && 4032 == image->shape()[3], true);
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 20);
// Manually terminate the pipeline
......@@ -628,9 +620,7 @@ TEST_F(MindDataTestPipeline, TestRandomSharpness) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestRandomAffineSuccess1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomAffineSuccess1 with non-default params.";
TEST_F(MindDataTestPipeline, TestRandomFlip) {
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
......@@ -642,12 +632,14 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess1) {
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> affine =
vision::RandomAffine({30.0, 30.0}, {0.0, 0.0}, {2.0, 2.0}, {10.0, 10.0, 20.0, 20.0});
EXPECT_NE(affine, nullptr);
std::shared_ptr<TensorOperation> random_vertical_flip_op = vision::RandomVerticalFlip(0.5);
EXPECT_NE(random_vertical_flip_op, nullptr);
std::shared_ptr<TensorOperation> random_horizontal_flip_op = vision::RandomHorizontalFlip(0.5);
EXPECT_NE(random_horizontal_flip_op, nullptr);
// Create a Map operation on ds
ds = ds->Map({affine});
ds = ds->Map({random_vertical_flip_op, random_horizontal_flip_op});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
......@@ -678,9 +670,7 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess1) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestRandomAffineSuccess2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomAffineSuccess2 with default params.";
TEST_F(MindDataTestPipeline, TestRandomRotation) {
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
......@@ -692,11 +682,11 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess2) {
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> affine = vision::RandomAffine({0.0, 0.0});
EXPECT_NE(affine, nullptr);
std::shared_ptr<TensorOperation> random_rotation_op = vision::RandomRotation({-180, 180});
EXPECT_NE(random_rotation_op, nullptr);
// Create a Map operation on ds
ds = ds->Map({affine});
ds = ds->Map({random_rotation_op});
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
......@@ -727,42 +717,32 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess2) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestRandomAffineFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomAffineFail with invalid params.";
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> affine = vision::RandomAffine({0.0, 0.0}, {});
EXPECT_EQ(affine, nullptr);
// Invalid number of values for translate
affine = vision::RandomAffine({0.0, 0.0}, {1, 1, 1, 1});
EXPECT_EQ(affine, nullptr);
// Invalid number of values for shear
affine = vision::RandomAffine({30.0, 30.0}, {0.0, 0.0}, {2.0, 2.0}, {10.0, 10.0});
EXPECT_EQ(affine, nullptr);
}
TEST_F(MindDataTestPipeline, TestRandomRotation) {
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
TEST_F(MindDataTestPipeline, TestUniformAugWithOps) {
// Create a Mnist Dataset
std::string folder_path = datasets_root_path_ + "/testMnistData/";
std::shared_ptr<Dataset> ds = Mnist(folder_path, RandomSampler(false, 20));
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
int32_t repeat_num = 2;
int32_t repeat_num = 1;
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops
std::shared_ptr<TensorOperation> random_rotation_op = vision::RandomRotation({-180, 180});
EXPECT_NE(random_rotation_op, nullptr);
std::shared_ptr<TensorOperation> resize_op = vision::Resize({30, 30});
EXPECT_NE(resize_op, nullptr);
// Create a Map operation on ds
ds = ds->Map({random_rotation_op});
EXPECT_NE(ds, nullptr);
std::shared_ptr<TensorOperation> random_crop_op = vision::RandomCrop({28, 28});
EXPECT_NE(random_crop_op, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
std::shared_ptr<TensorOperation> center_crop_op = vision::CenterCrop({16, 16});
EXPECT_NE(center_crop_op, nullptr);
std::shared_ptr<TensorOperation> uniform_aug_op = vision::UniformAugment({random_crop_op, center_crop_op}, 2);
EXPECT_NE(uniform_aug_op, nullptr);
// Create a Map operation on ds
ds = ds->Map({resize_op, uniform_aug_op});
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或