diff --git a/example/mnist_demo/lenet5_mnist_coverage.py b/example/mnist_demo/lenet5_mnist_coverage.py index 5f299dd4ad037a49fd7cc44ea4355671d90d852b..c497af698ce6140a8c0841a6c2bf21f24268bde9 100644 --- a/example/mnist_demo/lenet5_mnist_coverage.py +++ b/example/mnist_demo/lenet5_mnist_coverage.py @@ -51,7 +51,7 @@ def test_lenet_mnist_coverage(): train_images = np.concatenate(train_images, axis=0) # initialize fuzz test with training dataset - model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, train_images) + model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, train_images) # fuzz test with original test data # get test data diff --git a/example/mnist_demo/lenet5_mnist_fuzzing.py b/example/mnist_demo/lenet5_mnist_fuzzing.py index 273e5344cd8e6d45a23fbbf9d59fc3bb8655d75a..b08a15799faca97dc01350f962afb03a45d18653 100644 --- a/example/mnist_demo/lenet5_mnist_fuzzing.py +++ b/example/mnist_demo/lenet5_mnist_fuzzing.py @@ -39,14 +39,22 @@ def test_lenet_mnist_fuzzing(): load_param_into_net(net, load_dict) model = Model(net) mutate_config = [{'method': 'Blur', - 'params': {'auto_param': True}}, - {'method': 'Contrast', - 'params': {'factor': 2}}, - {'method': 'Translate', - 'params': {'x_bias': 0.1, 'y_bias': 0.2}}, - {'method': 'FGSM', - 'params': {'eps': 0.1, 'alpha': 0.1}} - ] + 'params': {'auto_param': True}}, + {'method': 'Contrast', + 'params': {'auto_param': True}}, + {'method': 'Translate', + 'params': {'auto_param': True}}, + {'method': 'Brightness', + 'params': {'auto_param': True}}, + {'method': 'Noise', + 'params': {'auto_param': True}}, + {'method': 'Scale', + 'params': {'auto_param': True}}, + {'method': 'Shear', + 'params': {'auto_param': True}}, + {'method': 'FGSM', + 'params': {'eps': 0.3, 'alpha': 0.1}} + ] # get training data data_list = "./MNIST_unzip/train" @@ -59,7 +67,7 @@ def test_lenet_mnist_fuzzing(): train_images = np.concatenate(train_images, axis=0) # initialize fuzz test with training dataset - model_coverage_test = ModelCoverageMetrics(model, 1000, 10, train_images) + model_coverage_test = ModelCoverageMetrics(model, 10, 1000, train_images) # fuzz test with original test data # get test data diff --git a/mindarmour/fuzzing/fuzzing.py b/mindarmour/fuzzing/fuzzing.py index af57bf278fb260a868d5e546fb0d9bff881194b7..36fa162287562d99d73f32d016349f2dcd8d20bd 100644 --- a/mindarmour/fuzzing/fuzzing.py +++ b/mindarmour/fuzzing/fuzzing.py @@ -102,8 +102,9 @@ class Fuzzer: self._target_model = check_model('model', target_model, Model) train_dataset = check_numpy_param('train_dataset', train_dataset) self._coverage_metrics = ModelCoverageMetrics(target_model, + neuron_num, segmented_num, - neuron_num, train_dataset) + train_dataset) # Allowed mutate strategies so far. self._strategies = {'Contrast': Contrast, 'Brightness': Brightness, 'Blur': Blur, 'Noise': Noise, 'Translate': Translate, @@ -190,11 +191,6 @@ class Fuzzer: eval_metrics_ = [] avaliable_metrics = ['accuracy', 'attack_success_rate', 'kmnc', 'nbc', 'snac'] for elem in eval_metrics: - if not isinstance(elem, str): - msg = 'the type of metric in list `eval_metrics` must be str, but got {}.' \ - .format(type(elem)) - LOGGER.error(TAG, msg) - raise TypeError(msg) if elem not in avaliable_metrics: msg = 'metric in list `eval_metrics` must be in {}, but got {}.' \ .format(avaliable_metrics, elem) diff --git a/mindarmour/fuzzing/model_coverage_metrics.py b/mindarmour/fuzzing/model_coverage_metrics.py index 1dbd6219805d8107a11972629f5102b8544d170e..5d2678e75677e5a694a3b6817d6fe8c84e06e163 100644 --- a/mindarmour/fuzzing/model_coverage_metrics.py +++ b/mindarmour/fuzzing/model_coverage_metrics.py @@ -43,8 +43,8 @@ class ModelCoverageMetrics: Args: model (Model): The pre-trained model which waiting for testing. - segmented_num (int): The number of segmented sections of neurons' output intervals. neuron_num (int): The number of testing neurons. + segmented_num (int): The number of segmented sections of neurons' output intervals. train_dataset (numpy.ndarray): Training dataset used for determine the neurons' output boundaries. @@ -56,14 +56,14 @@ class ModelCoverageMetrics: >>> train_images = np.random.random((10000, 1, 32, 32)).astype(np.float32) >>> test_images = np.random.random((5000, 1, 32, 32)).astype(np.float32) >>> model = Model(net) - >>> model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, train_images) + >>> model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, train_images) >>> model_fuzz_test.calculate_coverage(test_images) >>> print('KMNC of this test is : %s', model_fuzz_test.get_kmnc()) >>> print('NBC of this test is : %s', model_fuzz_test.get_nbc()) >>> print('SNAC of this test is : %s', model_fuzz_test.get_snac()) """ - def __init__(self, model, segmented_num, neuron_num, train_dataset): + def __init__(self, model, neuron_num, segmented_num, train_dataset): self._model = check_model('model', model, Model) self._segmented_num = check_int_positive('segmented_num', segmented_num) self._neuron_num = check_int_positive('neuron_num', neuron_num) diff --git a/tests/ut/python/fuzzing/test_coverage_metrics.py b/tests/ut/python/fuzzing/test_coverage_metrics.py index 7a69863c393b74bded83f97b9901da06308cd230..129080921a9785ecdb7bc3435863f9ff2f168e9e 100644 --- a/tests/ut/python/fuzzing/test_coverage_metrics.py +++ b/tests/ut/python/fuzzing/test_coverage_metrics.py @@ -71,7 +71,7 @@ def test_lenet_mnist_coverage_cpu(): # initialize fuzz test with training dataset training_data = (np.random.random((10000, 10))*20).astype(np.float32) - model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, training_data) + model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, training_data) # fuzz test with original test data # get test data @@ -105,7 +105,7 @@ def test_lenet_mnist_coverage_ascend(): # initialize fuzz test with training dataset training_data = (np.random.random((10000, 10))*20).astype(np.float32) - model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, training_data) + model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, training_data) # fuzz test with original test data # get test data diff --git a/tests/ut/python/fuzzing/test_fuzzer.py b/tests/ut/python/fuzzing/test_fuzzer.py index 60af20da01a8b38a84dc45f65b49524b77598b8f..fc6ca2687998b5da87ae3c7171371edec0753e95 100644 --- a/tests/ut/python/fuzzing/test_fuzzer.py +++ b/tests/ut/python/fuzzing/test_fuzzer.py @@ -102,7 +102,7 @@ def test_fuzzing_ascend(): ] # initialize fuzz test with training dataset train_images = np.random.rand(32, 1, 32, 32).astype(np.float32) - model_coverage_test = ModelCoverageMetrics(model, 1000, 10, train_images) + model_coverage_test = ModelCoverageMetrics(model, 10, 1000, train_images) # fuzz test with original test data # get test data @@ -148,7 +148,7 @@ def test_fuzzing_cpu(): ] # initialize fuzz test with training dataset train_images = np.random.rand(32, 1, 32, 32).astype(np.float32) - model_coverage_test = ModelCoverageMetrics(model, 1000, 10, train_images) + model_coverage_test = ModelCoverageMetrics(model, 10, 1000, train_images) # fuzz test with original test data # get test data