diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index 2d5c219b71bfb403f593b54124a79846314f92c1..db2b5169d2d2c72dbf503236de11eb553e9afa0e 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -38,7 +38,7 @@ from .iterators import DictIterator, TupleIterator from .validators import check, check_batch, check_shuffle, check_map, check_repeat, check_zip, check_rename, \ check_project, check_imagefolderdatasetv2, check_mnist_cifar_dataset, check_manifestdataset, \ check_tfrecorddataset, check_vocdataset, check_celebadataset, check_minddataset, check_generatordataset, \ - check_zip_dataset, check_add_column, check_columns + check_zip_dataset from ..core.datatypes import mstype_to_detype, mstypelist_to_detypelist try: @@ -2334,20 +2334,13 @@ class Schema: self.dataset_type = '' self.num_rows = 0 else: - if not os.path.isfile(schema_file) or not os.access(schema_file, os.R_OK): - raise ValueError("The file %s does not exist or permission denied!" % schema_file) try: with open(schema_file, 'r') as load_f: json_obj = json.load(load_f) + self.from_json(json_obj) except json.decoder.JSONDecodeError: - raise RuntimeError("Schema file failed to load.") - except UnicodeDecodeError: - raise RuntimeError("Schema file failed to decode.") - except Exception: - raise RuntimeError("Schema file failed to open.") - self.from_json(json_obj) + raise RuntimeError("Schema file failed to load") - @check_add_column def add_column(self, name, de_type, shape=None): """ Add new column to the schema. @@ -2366,8 +2359,10 @@ class Schema: if isinstance(de_type, typing.Type): de_type = mstype_to_detype(de_type) new_column["type"] = str(de_type) - else: + elif isinstance(de_type, str): new_column["type"] = str(DataType(de_type)) + else: + raise ValueError("Unknown column type") if shape is not None: new_column["shape"] = shape @@ -2404,7 +2399,6 @@ class Schema: RuntimeError: If column's name field is missing. RuntimeError: If column's type field is missing. """ - check_columns(columns, columns) self.columns = [] for col in columns: name = None @@ -2449,8 +2443,6 @@ class Schema: RuntimeError: if dataset type is missing in the object. RuntimeError: if columns are missing in the object. """ - if not isinstance(json_obj, dict) or json_obj is None: - raise ValueError("Expected non-empty dict.") for k, v in json_obj.items(): if k == "datasetType": self.dataset_type = v diff --git a/mindspore/dataset/engine/validators.py b/mindspore/dataset/engine/validators.py index 1c374ae879124a835558a49c7633fd1258d6746a..b4d22a4a013adcb23da23b58c713540179cc9b0f 100644 --- a/mindspore/dataset/engine/validators.py +++ b/mindspore/dataset/engine/validators.py @@ -19,15 +19,10 @@ import inspect as ins import os from functools import wraps from multiprocessing import cpu_count -from mindspore._c_expression import typing from . import samplers from . import datasets INT32_MAX = 2147483647 -valid_detype = [ - "bool", "int8", "int16", "int32", "int64", "uint8", "uint16", - "uint32", "uint64", "float16", "float32", "float64" -] def check(method): @@ -193,12 +188,6 @@ def check(method): return wrapper -def check_valid_detype(type_): - if type_ not in valid_detype: - raise ValueError("Unknown column type") - return True - - def check_filename(path): """ check the filename in the path @@ -754,41 +743,3 @@ def check_project(method): return method(*args, **kwargs) return new_method - - -def check_shape(shape, name): - if isinstance(shape, list): - for element in shape: - if not isinstance(element, int): - raise TypeError( - "Each element in {0} should be of type int. Got {1}.".format(name, type(element))) - else: - raise TypeError("Expected int list.") - - -def check_add_column(method): - """check the input arguments of add_column.""" - @wraps(method) - def new_method(*args, **kwargs): - param_dict = make_param_dict(method, args, kwargs) - - # check name; required argument - name = param_dict.get("name") - if not isinstance(name, str) or not name: - raise TypeError("Expected non-empty string.") - - # check type; required argument - de_type = param_dict.get("de_type") - if not isinstance(de_type, str) or not de_type: - raise TypeError("Expected non-empty string.") - if not isinstance(de_type, typing.Type) and not check_valid_detype(de_type): - raise ValueError("Unknown column type.") - - # check shape - shape = param_dict.get("shape") - if shape is not None: - check_shape(shape, "shape") - - return method(*args, **kwargs) - - return new_method diff --git a/tests/ut/python/ops/test_ops_check.py b/tests/ut/python/ops/test_ops_check.py index 5083878daeb8ca1e3d0c30b9a93e78b0dac9025c..a7e1b41c4a6989957e5effde11d6415e5c5c394c 100644 --- a/tests/ut/python/ops/test_ops_check.py +++ b/tests/ut/python/ops/test_ops_check.py @@ -62,7 +62,7 @@ def test_net_without_construct(): try: _executor.compile(net, inp) except RuntimeError as err: - if str(err).find("unsupported syntax 'Raise' at ") >= 0: + if str(err).find("Unsupported syntax 'Raise' at ") >= 0: print(str(err)) else: raise err @@ -86,7 +86,7 @@ def test_net_with_raise(): try: _executor.compile(net, inp) except RuntimeError as err: - if str(err).find("unsupported syntax 'Raise' at ") >= 0: + if str(err).find("Unsupported syntax 'Raise' at ") >= 0: print(str(err)) else: raise err diff --git a/tests/ut/python/runtest.sh b/tests/ut/python/runtest.sh index 035e454f246776534e318a88b2811a37d516fc2d..6502d3a1bffb1bb300b06fa96e27b53637d47be3 100755 --- a/tests/ut/python/runtest.sh +++ b/tests/ut/python/runtest.sh @@ -40,12 +40,8 @@ fi if [ $# -gt 0 ]; then pytest -s --ignore=$1/pynative_mode --ignore=$1/parallel --ignore=$1/train $IGNORE_EXEC $1 - pytest -n 4 --dist=loadfile -v $1/parallel - pytest -n 4 --dist=loadfile -v $1/train else pytest --ignore=$CURRPATH/pynative_mode --ignore=$CURRPATH/parallel --ignore=$CURRPATH/train $IGNORE_EXEC $CURRPATH - pytest -n 4 --dist=loadfile -v $CURRPATH/parallel - pytest -n 4 --dist=loadfile -v $CURRPATH/train fi RET=$? @@ -57,6 +53,18 @@ if [ ${RET} -ne 0 ]; then exit ${RET} fi +if [ $# -gt 0 ]; then + pytest -n 4 --dist=loadfile -v $1/parallel $1/train +else + pytest -n 4 --dist=loadfile -v $CURRPATH/parallel $CURRPATH/train +fi + +RET=$? + +if [ ${RET} -ne 0 ]; then + exit ${RET} +fi + if [ $# -gt 0 ]; then pytest -s $1/pynative_mode else