!91 fix issues

Merge pull request !91 from ZhidanLiu/master

example/mnist_demo/lenet5_mnist_coverage.py

@@ -51,7 +51,7 @@ def test_lenet_mnist_coverage():
     train_images = np.concatenate(train_images, axis=0)
 
     # initialize fuzz test with training dataset
-    model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, train_images)
+    model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, train_images)
 
     # fuzz test with original test data
     # get test data

example/mnist_demo/lenet5_mnist_fuzzing.py

@@ -41,11 +41,19 @@ def test_lenet_mnist_fuzzing():
     mutate_config = [{'method': 'Blur',
                      'params': {'auto_param': True}},
                     {'method': 'Contrast',
-                      'params': {'factor': 2}},
+                     'params': {'auto_param': True}},
                     {'method': 'Translate',
-                      'params': {'x_bias': 0.1, 'y_bias': 0.2}},
+                     'params': {'auto_param': True}},
+                    {'method': 'Brightness',
+                     'params': {'auto_param': True}},
+                    {'method': 'Noise',
+                     'params': {'auto_param': True}},
+                    {'method': 'Scale',
+                     'params': {'auto_param': True}},
+                    {'method': 'Shear',
+                     'params': {'auto_param': True}},
                     {'method': 'FGSM',
-                      'params': {'eps': 0.1, 'alpha': 0.1}}
+                     'params': {'eps': 0.3, 'alpha': 0.1}}
                     ]
 
     # get training data
@@ -59,7 +67,7 @@ def test_lenet_mnist_fuzzing():
     train_images = np.concatenate(train_images, axis=0)
 
     # initialize fuzz test with training dataset
-    model_coverage_test = ModelCoverageMetrics(model, 1000, 10, train_images)
+    model_coverage_test = ModelCoverageMetrics(model, 10, 1000, train_images)
 
     # fuzz test with original test data
     # get test data

mindarmour/fuzzing/fuzzing.py

@@ -102,8 +102,9 @@ class Fuzzer:
         self._target_model = check_model('model', target_model, Model)
         train_dataset = check_numpy_param('train_dataset', train_dataset)
         self._coverage_metrics = ModelCoverageMetrics(target_model,
+                                                      neuron_num,
                                                       segmented_num,
-                                                      neuron_num, train_dataset)
+                                                      train_dataset)
         # Allowed mutate strategies so far.
         self._strategies = {'Contrast': Contrast, 'Brightness': Brightness,
                             'Blur': Blur, 'Noise': Noise, 'Translate': Translate,
@@ -190,11 +191,6 @@ class Fuzzer:
             eval_metrics_ = []
             avaliable_metrics = ['accuracy', 'attack_success_rate', 'kmnc', 'nbc', 'snac']
             for elem in eval_metrics:
-                if not isinstance(elem, str):
-                    msg = 'the type of metric in list `eval_metrics` must be str, but got {}.' \
-                        .format(type(elem))
-                    LOGGER.error(TAG, msg)
-                    raise TypeError(msg)
                 if elem not in avaliable_metrics:
                     msg = 'metric in list `eval_metrics` must be in {}, but got {}.' \
                         .format(avaliable_metrics, elem)

mindarmour/fuzzing/model_coverage_metrics.py

@@ -43,8 +43,8 @@ class ModelCoverageMetrics:
 
     Args:
         model (Model): The pre-trained model which waiting for testing.
-        segmented_num (int): The number of segmented sections of neurons' output intervals.
         neuron_num (int): The number of testing neurons.
+        segmented_num (int): The number of segmented sections of neurons' output intervals.
         train_dataset (numpy.ndarray): Training dataset used for determine
             the neurons' output boundaries.
 
@@ -56,14 +56,14 @@ class ModelCoverageMetrics:
         >>> train_images = np.random.random((10000, 1, 32, 32)).astype(np.float32)
         >>> test_images = np.random.random((5000, 1, 32, 32)).astype(np.float32)
         >>> model = Model(net)
-        >>> model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, train_images)
+        >>> model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, train_images)
         >>> model_fuzz_test.calculate_coverage(test_images)
         >>> print('KMNC of this test is : %s', model_fuzz_test.get_kmnc())
         >>> print('NBC of this test is : %s', model_fuzz_test.get_nbc())
         >>> print('SNAC of this test is : %s', model_fuzz_test.get_snac())
     """
 
-    def __init__(self, model, segmented_num, neuron_num, train_dataset):
+    def __init__(self, model, neuron_num, segmented_num, train_dataset):
         self._model = check_model('model', model, Model)
         self._segmented_num = check_int_positive('segmented_num', segmented_num)
         self._neuron_num = check_int_positive('neuron_num', neuron_num)

tests/ut/python/fuzzing/test_coverage_metrics.py

@@ -71,7 +71,7 @@ def test_lenet_mnist_coverage_cpu():
 
     # initialize fuzz test with training dataset
     training_data = (np.random.random((10000, 10))*20).astype(np.float32)
-    model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, training_data)
+    model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, training_data)
 
     # fuzz test with original test data
     # get test data
@@ -105,7 +105,7 @@ def test_lenet_mnist_coverage_ascend():
 
     # initialize fuzz test with training dataset
     training_data = (np.random.random((10000, 10))*20).astype(np.float32)
-    model_fuzz_test = ModelCoverageMetrics(model, 10000, 10, training_data)
+    model_fuzz_test = ModelCoverageMetrics(model, 10, 1000, training_data)
 
     # fuzz test with original test data
     # get test data

tests/ut/python/fuzzing/test_fuzzer.py

@@ -102,7 +102,7 @@ def test_fuzzing_ascend():
                      ]
     # initialize fuzz test with training dataset
     train_images = np.random.rand(32, 1, 32, 32).astype(np.float32)
-    model_coverage_test = ModelCoverageMetrics(model, 1000, 10, train_images)
+    model_coverage_test = ModelCoverageMetrics(model, 10, 1000, train_images)
 
     # fuzz test with original test data
     # get test data
@@ -148,7 +148,7 @@ def test_fuzzing_cpu():
                      ]
     # initialize fuzz test with training dataset
     train_images = np.random.rand(32, 1, 32, 32).astype(np.float32)
-    model_coverage_test = ModelCoverageMetrics(model, 1000, 10, train_images)
+    model_coverage_test = ModelCoverageMetrics(model, 10, 1000, train_images)
 
     # fuzz test with original test data
     # get test data