From afa3f75fc349161bec0bee968f5b850e56d6991d Mon Sep 17 00:00:00 2001 From: ZhidanLiu Date: Mon, 24 Aug 2020 17:10:42 +0800 Subject: [PATCH] fix issues --- example/mnist_demo/lenet5_mnist_coverage.py | 2 +- example/mnist_demo/lenet5_mnist_fuzzing.py | 26 ++++++++++++------- mindarmour/fuzzing/fuzzing.py | 8 ++---- mindarmour/fuzzing/model_coverage_metrics.py | 6 ++--- .../python/fuzzing/test_coverage_metrics.py | 4 +-- tests/ut/python/fuzzing/test_fuzzer.py | 4 +-- 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/example/mnist_demo/lenet5_mnist_coverage.py b/example/mnist_demo/lenet5_mnist_coverage.py index 5f299dd..c497af6 100644 --- a/example/mnist_demo/lenet5_mnist_coverage.py +++ b/example/mnist_demo/lenet5_mnist_coverage.py @@ -51,7 +51,7 @@ def test_lenet_mnist_coverage(): train_images = np.concatenate(train_images, axis=0) # initialize fuzz test with training dataset - model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, train_images) + model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, train_images) # fuzz test with original test data # get test data diff --git a/example/mnist_demo/lenet5_mnist_fuzzing.py b/example/mnist_demo/lenet5_mnist_fuzzing.py index 273e534..b08a157 100644 --- a/example/mnist_demo/lenet5_mnist_fuzzing.py +++ b/example/mnist_demo/lenet5_mnist_fuzzing.py @@ -39,14 +39,22 @@ def test_lenet_mnist_fuzzing(): load_param_into_net(net, load_dict) model = Model(net) mutate_config = [{'method': 'Blur', - 'params': {'auto_param': True}}, - {'method': 'Contrast', - 'params': {'factor': 2}}, - {'method': 'Translate', - 'params': {'x_bias': 0.1, 'y_bias': 0.2}}, - {'method': 'FGSM', - 'params': {'eps': 0.1, 'alpha': 0.1}} - ] + 'params': {'auto_param': True}}, + {'method': 'Contrast', + 'params': {'auto_param': True}}, + {'method': 'Translate', + 'params': {'auto_param': True}}, + {'method': 'Brightness', + 'params': {'auto_param': True}}, + {'method': 'Noise', + 'params': {'auto_param': True}}, + {'method': 'Scale', + 'params': {'auto_param': True}}, + {'method': 'Shear', + 'params': {'auto_param': True}}, + {'method': 'FGSM', + 'params': {'eps': 0.3, 'alpha': 0.1}} + ] # get training data data_list = "./MNIST_unzip/train" @@ -59,7 +67,7 @@ def test_lenet_mnist_fuzzing(): train_images = np.concatenate(train_images, axis=0) # initialize fuzz test with training dataset - model_coverage_test = ModelCoverageMetrics(model, 1000, 10, train_images) + model_coverage_test = ModelCoverageMetrics(model, 10, 1000, train_images) # fuzz test with original test data # get test data diff --git a/mindarmour/fuzzing/fuzzing.py b/mindarmour/fuzzing/fuzzing.py index af57bf2..36fa162 100644 --- a/mindarmour/fuzzing/fuzzing.py +++ b/mindarmour/fuzzing/fuzzing.py @@ -102,8 +102,9 @@ class Fuzzer: self._target_model = check_model('model', target_model, Model) train_dataset = check_numpy_param('train_dataset', train_dataset) self._coverage_metrics = ModelCoverageMetrics(target_model, + neuron_num, segmented_num, - neuron_num, train_dataset) + train_dataset) # Allowed mutate strategies so far. self._strategies = {'Contrast': Contrast, 'Brightness': Brightness, 'Blur': Blur, 'Noise': Noise, 'Translate': Translate, @@ -190,11 +191,6 @@ class Fuzzer: eval_metrics_ = [] avaliable_metrics = ['accuracy', 'attack_success_rate', 'kmnc', 'nbc', 'snac'] for elem in eval_metrics: - if not isinstance(elem, str): - msg = 'the type of metric in list `eval_metrics` must be str, but got {}.' \ - .format(type(elem)) - LOGGER.error(TAG, msg) - raise TypeError(msg) if elem not in avaliable_metrics: msg = 'metric in list `eval_metrics` must be in {}, but got {}.' \ .format(avaliable_metrics, elem) diff --git a/mindarmour/fuzzing/model_coverage_metrics.py b/mindarmour/fuzzing/model_coverage_metrics.py index 1dbd621..5d2678e 100644 --- a/mindarmour/fuzzing/model_coverage_metrics.py +++ b/mindarmour/fuzzing/model_coverage_metrics.py @@ -43,8 +43,8 @@ class ModelCoverageMetrics: Args: model (Model): The pre-trained model which waiting for testing. - segmented_num (int): The number of segmented sections of neurons' output intervals. neuron_num (int): The number of testing neurons. + segmented_num (int): The number of segmented sections of neurons' output intervals. train_dataset (numpy.ndarray): Training dataset used for determine the neurons' output boundaries. @@ -56,14 +56,14 @@ class ModelCoverageMetrics: >>> train_images = np.random.random((10000, 1, 32, 32)).astype(np.float32) >>> test_images = np.random.random((5000, 1, 32, 32)).astype(np.float32) >>> model = Model(net) - >>> model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, train_images) + >>> model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, train_images) >>> model_fuzz_test.calculate_coverage(test_images) >>> print('KMNC of this test is : %s', model_fuzz_test.get_kmnc()) >>> print('NBC of this test is : %s', model_fuzz_test.get_nbc()) >>> print('SNAC of this test is : %s', model_fuzz_test.get_snac()) """ - def __init__(self, model, segmented_num, neuron_num, train_dataset): + def __init__(self, model, neuron_num, segmented_num, train_dataset): self._model = check_model('model', model, Model) self._segmented_num = check_int_positive('segmented_num', segmented_num) self._neuron_num = check_int_positive('neuron_num', neuron_num) diff --git a/tests/ut/python/fuzzing/test_coverage_metrics.py b/tests/ut/python/fuzzing/test_coverage_metrics.py index 7a69863..1290809 100644 --- a/tests/ut/python/fuzzing/test_coverage_metrics.py +++ b/tests/ut/python/fuzzing/test_coverage_metrics.py @@ -71,7 +71,7 @@ def test_lenet_mnist_coverage_cpu(): # initialize fuzz test with training dataset training_data = (np.random.random((10000, 10))*20).astype(np.float32) - model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, training_data) + model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, training_data) # fuzz test with original test data # get test data @@ -105,7 +105,7 @@ def test_lenet_mnist_coverage_ascend(): # initialize fuzz test with training dataset training_data = (np.random.random((10000, 10))*20).astype(np.float32) - model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, training_data) + model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, training_data) # fuzz test with original test data # get test data diff --git a/tests/ut/python/fuzzing/test_fuzzer.py b/tests/ut/python/fuzzing/test_fuzzer.py index 60af20d..fc6ca26 100644 --- a/tests/ut/python/fuzzing/test_fuzzer.py +++ b/tests/ut/python/fuzzing/test_fuzzer.py @@ -102,7 +102,7 @@ def test_fuzzing_ascend(): ] # initialize fuzz test with training dataset train_images = np.random.rand(32, 1, 32, 32).astype(np.float32) - model_coverage_test = ModelCoverageMetrics(model, 1000, 10, train_images) + model_coverage_test = ModelCoverageMetrics(model, 10, 1000, train_images) # fuzz test with original test data # get test data @@ -148,7 +148,7 @@ def test_fuzzing_cpu(): ] # initialize fuzz test with training dataset train_images = np.random.rand(32, 1, 32, 32).astype(np.float32) - model_coverage_test = ModelCoverageMetrics(model, 1000, 10, train_images) + model_coverage_test = ModelCoverageMetrics(model, 10, 1000, train_images) # fuzz test with original test data # get test data -- GitLab