diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc index 522c3c111677433a78e8f0418161f7519e9ac29b..51237f58cd17de4a376170e044ec556a2cdc1634 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc @@ -387,6 +387,9 @@ uint32_t DatasetOp::GenerateCRC(const std::shared_ptr &op) { ss_str = std::regex_replace(ss_str, std::regex("Num workers.*\n"), ""); ss_str = std::regex_replace(ss_str, std::regex("\\[workers.*\\]"), ""); + // Filter out Number of rows when generating the check sum + ss_str = std::regex_replace(ss_str, std::regex("Number of rows.*\n"), ""); + // Filter out the Operator control flags field when generating the check sum ss_str = std::regex_replace(ss_str, std::regex("Operator control flags.*\n"), ""); diff --git a/tests/ut/python/dataset/test_cache_map.py b/tests/ut/python/dataset/test_cache_map.py index 5de969db6df7f74b4a6370f83b9f742c5733620a..72f33be576211ad30a23a40139180a1bd86f80c1 100644 --- a/tests/ut/python/dataset/test_cache_map.py +++ b/tests/ut/python/dataset/test_cache_map.py @@ -104,11 +104,11 @@ def test_cache_map_basic3(): decode_op = c_vision.Decode() ds1 = ds1.repeat(4) ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache) - print("ds1.dataset_size is ", ds1.get_dataset_size()) + logger.info("ds1.dataset_size is ", ds1.get_dataset_size()) num_iter = 0 for _ in ds1.create_dict_iterator(): - print("get data from dataset") + logger.info("get data from dataset") num_iter += 1 logger.info("Number of data in ds1: {} ".format(num_iter)) @@ -116,6 +116,31 @@ def test_cache_map_basic3(): logger.info('test_cache_basic3 Ended.\n') +def test_cache_map_basic4(): + """ + Test different rows result in core dump + """ + logger.info("Test cache basic 4") + some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True) + + # This DATA_DIR only has 2 images in it + ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache) + decode_op = c_vision.Decode() + ds1 = ds1.repeat(4) + ds1 = ds1.map(input_columns=["image"], operations=decode_op) + logger.info("ds1.dataset_size is ", ds1.get_dataset_size()) + shape = ds1.output_shapes() + + num_iter = 0 + for _ in ds1.create_dict_iterator(): + logger.info("get data from dataset") + num_iter += 1 + + logger.info("Number of data in ds1: {} ".format(num_iter)) + assert num_iter == 8 + logger.info('test_cache_basic3 Ended.\n') + + def test_cache_map_failure1(): """ Test nested cache (failure) @@ -154,10 +179,12 @@ def test_cache_map_failure1(): if __name__ == '__main__': test_cache_map_basic1() - print("test_cache_map_basic1 success.") + logger.info("test_cache_map_basic1 success.") test_cache_map_basic2() - print("test_cache_map_basic2 success.") + logger.info("test_cache_map_basic2 success.") test_cache_map_basic3() - print("test_cache_map_basic3 success.") + logger.info("test_cache_map_basic3 success.") + test_cache_map_basic4() + logger.info("test_cache_map_basic3 success.") test_cache_map_failure1() - print("test_cache_map_failure1 success.") + logger.info("test_cache_map_failure1 success.")