diff --git a/tests/ut/__init__.py b/tests/ut/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1fec660695e211c236f0e3d252b54730c6e8f53b --- /dev/null +++ b/tests/ut/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for developed features of MindArmour. +""" diff --git a/tests/ut/python/__init__.py b/tests/ut/python/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..190835ec18d96820a9bd8702bfa2a02dc915ffa0 --- /dev/null +++ b/tests/ut/python/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes python unit tests for developed features of MindArmour. +""" diff --git a/tests/ut/python/adv_robustness/__init__.py b/tests/ut/python/adv_robustness/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..02fa417b377dbf3995930ee95014e8e1673ed80a --- /dev/null +++ b/tests/ut/python/adv_robustness/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for model attacks, model defenses and +their evaluation functions. +""" diff --git a/tests/ut/python/adv_robustness/attacks/__init__.py b/tests/ut/python/adv_robustness/attacks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7c32a3086a2b61777b507dc67333b27556a514ad --- /dev/null +++ b/tests/ut/python/adv_robustness/attacks/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for white-box attack algorithms. +""" diff --git a/tests/ut/python/adv_robustness/attacks/black/__init__.py b/tests/ut/python/adv_robustness/attacks/black/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a0256d6c0727fb3ba90d3cb934946ceaf7623279 --- /dev/null +++ b/tests/ut/python/adv_robustness/attacks/black/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for black-box attack algorithms. +""" diff --git a/tests/ut/python/attacks/black/test_genetic_attack.py b/tests/ut/python/adv_robustness/attacks/black/test_genetic_attack.py similarity index 100% rename from tests/ut/python/attacks/black/test_genetic_attack.py rename to tests/ut/python/adv_robustness/attacks/black/test_genetic_attack.py diff --git a/tests/ut/python/attacks/black/test_hsja.py b/tests/ut/python/adv_robustness/attacks/black/test_hsja.py similarity index 90% rename from tests/ut/python/attacks/black/test_hsja.py rename to tests/ut/python/adv_robustness/attacks/black/test_hsja.py index b19e2032667dcc00ab91ecdcd879fe471717b913..d4ddaee277673df6b9966d46ceb145fb0df426e0 100644 --- a/tests/ut/python/attacks/black/test_hsja.py +++ b/tests/ut/python/adv_robustness/attacks/black/test_hsja.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import sys import numpy as np import pytest @@ -24,9 +23,7 @@ from mindarmour import BlackModel from mindarmour.adv_robustness.attacks import HopSkipJumpAttack from mindarmour.utils.logger import LogUtil -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), - "../../../../../")) -from example.mnist_demo.lenet5_net import LeNet5 +from ut.python.utils.mock_net import Net context.set_context(mode=context.GRAPH_MODE) context.set_context(device_target="Ascend") @@ -64,25 +61,27 @@ def random_target_labels(true_labels): def create_target_images(dataset, data_labels, target_labels): res = [] for label in target_labels: - for i in range(len(data_labels)): - if data_labels[i] == label: + for i, data_label in enumerate(data_labels): + if data_label == label: res.append(dataset[i]) break return np.array(res) + # public variable def get_model(): # upload trained network current_dir = os.path.dirname(os.path.abspath(__file__)) ckpt_name = os.path.join(current_dir, - '../../test_data/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') - net = LeNet5() + '../../../dataset/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') + net = Net() load_dict = load_checkpoint(ckpt_name) load_param_into_net(net, load_dict) net.set_train(False) model = ModelToBeAttacked(net) return model + @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_ascend_training @@ -97,9 +96,9 @@ def test_hsja_mnist_attack(): # get test data test_images_set = np.load(os.path.join(current_dir, - '../../test_data/test_images.npy')) + '../../../dataset/test_images.npy')) test_labels_set = np.load(os.path.join(current_dir, - '../../test_data/test_labels.npy')) + '../../../dataset/test_labels.npy')) # prediction accuracy before attack model = get_model() batch_num = 1 # the number of batches of attacking samples diff --git a/tests/ut/python/attacks/black/test_nes.py b/tests/ut/python/adv_robustness/attacks/black/test_nes.py similarity index 92% rename from tests/ut/python/attacks/black/test_nes.py rename to tests/ut/python/adv_robustness/attacks/black/test_nes.py index 6a849209d859224dbc04befa96507aaf760474f0..d9fa516a6e20bba5492763332490b7424111b86b 100644 --- a/tests/ut/python/attacks/black/test_nes.py +++ b/tests/ut/python/adv_robustness/attacks/black/test_nes.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import sys import numpy as np import pytest @@ -24,9 +23,7 @@ from mindarmour import BlackModel from mindarmour.adv_robustness.attacks import NES from mindarmour.utils.logger import LogUtil -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), - "../../../../../")) -from example.mnist_demo.lenet5_net import LeNet5 +from ut.python.utils.mock_net import Net context.set_context(mode=context.GRAPH_MODE) context.set_context(device_target="Ascend") @@ -73,31 +70,35 @@ def _pseudorandom_target(index, total_indices, true_class): def create_target_images(dataset, data_labels, target_labels): res = [] for label in target_labels: - for i in range(len(data_labels)): - if data_labels[i] == label: + for i, data_label in enumerate(data_labels): + if data_label == label: res.append(dataset[i]) break return np.array(res) + def get_model(current_dir): ckpt_name = os.path.join(current_dir, - '../../test_data/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') - net = LeNet5() + '../../../dataset/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') + net = Net() load_dict = load_checkpoint(ckpt_name) load_param_into_net(net, load_dict) net.set_train(False) model = ModelToBeAttacked(net) return model + def get_dataset(current_dir): # upload trained network # get test data test_images = np.load(os.path.join(current_dir, - '../../test_data/test_images.npy')) + '../../../dataset/test_images.npy')) test_labels = np.load(os.path.join(current_dir, - '../../test_data/test_labels.npy')) + '../../../dataset/test_labels.npy')) return test_images, test_labels + + def nes_mnist_attack(scene, top_k): """ hsja-Attack test diff --git a/tests/ut/python/attacks/black/test_pointwise_attack.py b/tests/ut/python/adv_robustness/attacks/black/test_pointwise_attack.py similarity index 86% rename from tests/ut/python/attacks/black/test_pointwise_attack.py rename to tests/ut/python/adv_robustness/attacks/black/test_pointwise_attack.py index 28c2c061add1161efc4f250fecc77f43cd00b580..6428cdb144b8e2122debb2fab1ad3a50800d895c 100644 --- a/tests/ut/python/attacks/black/test_pointwise_attack.py +++ b/tests/ut/python/adv_robustness/attacks/black/test_pointwise_attack.py @@ -15,7 +15,6 @@ PointWise Attack test """ import os -import sys import numpy as np import pytest @@ -27,10 +26,7 @@ from mindarmour import BlackModel from mindarmour.adv_robustness.attacks import PointWiseAttack from mindarmour.utils.logger import LogUtil -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), - "../../../../../")) -from example.mnist_demo.lenet5_net import LeNet5 - +from ut.python.utils.mock_net import Net context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") @@ -65,16 +61,16 @@ def test_pointwise_attack_method(): # upload trained network current_dir = os.path.dirname(os.path.abspath(__file__)) ckpt_name = os.path.join(current_dir, - '../../test_data/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') - net = LeNet5() + '../../../dataset/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') + net = Net() load_dict = load_checkpoint(ckpt_name) load_param_into_net(net, load_dict) # get one mnist image input_np = np.load(os.path.join(current_dir, - '../../test_data/test_images.npy'))[:3] + '../../../dataset/test_images.npy'))[:3] labels = np.load(os.path.join(current_dir, - '../../test_data/test_labels.npy'))[:3] + '../../../dataset/test_labels.npy'))[:3] model = ModelToBeAttacked(net) pre_label = np.argmax(model.predict(input_np), axis=1) LOGGER.info(TAG, 'original sample predict labels are :{}'.format(pre_label)) diff --git a/tests/ut/python/attacks/black/test_pso_attack.py b/tests/ut/python/adv_robustness/attacks/black/test_pso_attack.py similarity index 100% rename from tests/ut/python/attacks/black/test_pso_attack.py rename to tests/ut/python/adv_robustness/attacks/black/test_pso_attack.py diff --git a/tests/ut/python/attacks/black/test_salt_and_pepper_attack.py b/tests/ut/python/adv_robustness/attacks/black/test_salt_and_pepper_attack.py similarity index 98% rename from tests/ut/python/attacks/black/test_salt_and_pepper_attack.py rename to tests/ut/python/adv_robustness/attacks/black/test_salt_and_pepper_attack.py index ad08b1b6ad537b2e0c093c4fee77bde831dfece8..28b4743c5ba9ee1f720079d0d7623a741a9999cc 100644 --- a/tests/ut/python/attacks/black/test_salt_and_pepper_attack.py +++ b/tests/ut/python/adv_robustness/attacks/black/test_salt_and_pepper_attack.py @@ -90,7 +90,7 @@ def test_salt_and_pepper_attack_method(): labels = labels.astype(np.float32) attack = SaltAndPepperNoiseAttack(model, sparse=False) - is_adv, adv_data, query_times = attack.generate(inputs, labels) + _, adv_data, _ = attack.generate(inputs, labels) assert np.any(adv_data[0] != inputs[0]), 'Salt and pepper attack method: ' \ 'generate value must not be equal' \ ' to original value.' diff --git a/tests/ut/python/attacks/test_batch_generate_attack.py b/tests/ut/python/adv_robustness/attacks/test_batch_generate_attack.py similarity index 100% rename from tests/ut/python/attacks/test_batch_generate_attack.py rename to tests/ut/python/adv_robustness/attacks/test_batch_generate_attack.py diff --git a/tests/ut/python/attacks/test_cw.py b/tests/ut/python/adv_robustness/attacks/test_cw.py similarity index 100% rename from tests/ut/python/attacks/test_cw.py rename to tests/ut/python/adv_robustness/attacks/test_cw.py diff --git a/tests/ut/python/attacks/test_deep_fool.py b/tests/ut/python/adv_robustness/attacks/test_deep_fool.py similarity index 100% rename from tests/ut/python/attacks/test_deep_fool.py rename to tests/ut/python/adv_robustness/attacks/test_deep_fool.py diff --git a/tests/ut/python/attacks/test_gradient_method.py b/tests/ut/python/adv_robustness/attacks/test_gradient_method.py similarity index 100% rename from tests/ut/python/attacks/test_gradient_method.py rename to tests/ut/python/adv_robustness/attacks/test_gradient_method.py diff --git a/tests/ut/python/attacks/test_iterative_gradient_method.py b/tests/ut/python/adv_robustness/attacks/test_iterative_gradient_method.py similarity index 100% rename from tests/ut/python/attacks/test_iterative_gradient_method.py rename to tests/ut/python/adv_robustness/attacks/test_iterative_gradient_method.py diff --git a/tests/ut/python/attacks/test_jsma.py b/tests/ut/python/adv_robustness/attacks/test_jsma.py similarity index 100% rename from tests/ut/python/attacks/test_jsma.py rename to tests/ut/python/adv_robustness/attacks/test_jsma.py diff --git a/tests/ut/python/attacks/test_lbfgs.py b/tests/ut/python/adv_robustness/attacks/test_lbfgs.py similarity index 82% rename from tests/ut/python/attacks/test_lbfgs.py rename to tests/ut/python/adv_robustness/attacks/test_lbfgs.py index 097df4e014f5bcc92d4f5d191d131b02b25ed5f0..46e7f0551852c82446dcb432cca1202507802e62 100644 --- a/tests/ut/python/attacks/test_lbfgs.py +++ b/tests/ut/python/adv_robustness/attacks/test_lbfgs.py @@ -15,7 +15,6 @@ LBFGS-Attack test. """ import os -import sys import numpy as np import pytest @@ -25,9 +24,7 @@ from mindspore.train.serialization import load_checkpoint, load_param_into_net from mindarmour.adv_robustness.attacks import LBFGS from mindarmour.utils.logger import LogUtil -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), - "../../../../")) -from example.mnist_demo.lenet5_net import LeNet5 +from ut.python.utils.mock_net import Net context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") @@ -50,16 +47,16 @@ def test_lbfgs_attack(): # upload trained network current_dir = os.path.dirname(os.path.abspath(__file__)) ckpt_name = os.path.join(current_dir, - '../test_data/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') - net = LeNet5() + '../../dataset/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') + net = Net() load_dict = load_checkpoint(ckpt_name) load_param_into_net(net, load_dict) # get one mnist image input_np = np.load(os.path.join(current_dir, - '../test_data/test_images.npy'))[:1] + '../../dataset/test_images.npy'))[:1] label_np = np.load(os.path.join(current_dir, - '../test_data/test_labels.npy'))[:1] + '../../dataset/test_labels.npy'))[:1] LOGGER.debug(TAG, 'true label is :{}'.format(label_np[0])) classes = 10 target_np = np.random.randint(0, classes, 1) diff --git a/tests/ut/python/adv_robustness/defenses/__init__.py b/tests/ut/python/adv_robustness/defenses/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..93f240d2b8e51659ada297f9366cc219a6c96084 --- /dev/null +++ b/tests/ut/python/adv_robustness/defenses/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for defense algorithms. +""" diff --git a/tests/ut/python/defenses/test_ad.py b/tests/ut/python/adv_robustness/defenses/test_ad.py similarity index 98% rename from tests/ut/python/defenses/test_ad.py rename to tests/ut/python/adv_robustness/defenses/test_ad.py index d13b809965f914671157472694792da1e53267cf..2c0470df2f89945e3a7c3a1bdb541694caccb811 100644 --- a/tests/ut/python/defenses/test_ad.py +++ b/tests/ut/python/adv_robustness/defenses/test_ad.py @@ -23,10 +23,11 @@ from mindspore import context from mindspore import nn from mindspore.nn.optim.momentum import Momentum -from mock_net import Net from mindarmour.adv_robustness.defenses import AdversarialDefense from mindarmour.utils.logger import LogUtil +from ut.python.utils.mock_net import Net + LOGGER = LogUtil.get_instance() TAG = 'Ad_Test' diff --git a/tests/ut/python/defenses/test_ead.py b/tests/ut/python/adv_robustness/defenses/test_ead.py similarity index 98% rename from tests/ut/python/defenses/test_ead.py rename to tests/ut/python/adv_robustness/defenses/test_ead.py index ecf9dfa6875cf7cffa281de4cca24c051475a437..eb057e79d7fd8af1c072e93871e51352f9ae98e7 100644 --- a/tests/ut/python/defenses/test_ead.py +++ b/tests/ut/python/adv_robustness/defenses/test_ead.py @@ -22,13 +22,14 @@ from mindspore import context from mindspore import nn from mindspore.nn.optim.momentum import Momentum -from mock_net import Net from mindarmour.adv_robustness.attacks import FastGradientSignMethod from mindarmour.adv_robustness.attacks import \ ProjectedGradientDescent from mindarmour.adv_robustness.defenses import EnsembleAdversarialDefense from mindarmour.utils.logger import LogUtil +from ut.python.utils.mock_net import Net + LOGGER = LogUtil.get_instance() TAG = 'Ead_Test' diff --git a/tests/ut/python/defenses/test_nad.py b/tests/ut/python/adv_robustness/defenses/test_nad.py similarity index 97% rename from tests/ut/python/defenses/test_nad.py rename to tests/ut/python/adv_robustness/defenses/test_nad.py index c61fef8cf707a059341363a0b84fecef12994e5a..8c0e5e030deeb374de1061cb6e891f81c912e7ec 100644 --- a/tests/ut/python/defenses/test_nad.py +++ b/tests/ut/python/adv_robustness/defenses/test_nad.py @@ -22,10 +22,11 @@ from mindspore import context from mindspore import nn from mindspore.nn.optim.momentum import Momentum -from mock_net import Net from mindarmour.adv_robustness.defenses import NaturalAdversarialDefense from mindarmour.utils.logger import LogUtil +from ut.python.utils.mock_net import Net + LOGGER = LogUtil.get_instance() TAG = 'Nad_Test' diff --git a/tests/ut/python/defenses/test_pad.py b/tests/ut/python/adv_robustness/defenses/test_pad.py similarity index 98% rename from tests/ut/python/defenses/test_pad.py rename to tests/ut/python/adv_robustness/defenses/test_pad.py index c8ea77d77d519ab594f960428aa25d469ff4edee..fb4ed3a388d03522d8c589cfc47760c50c6f37b4 100644 --- a/tests/ut/python/defenses/test_pad.py +++ b/tests/ut/python/adv_robustness/defenses/test_pad.py @@ -22,10 +22,11 @@ from mindspore import context from mindspore import nn from mindspore.nn.optim.momentum import Momentum -from mock_net import Net from mindarmour.adv_robustness.defenses import ProjectedAdversarialDefense from mindarmour.utils.logger import LogUtil +from ut.python.utils.mock_net import Net + LOGGER = LogUtil.get_instance() TAG = 'Pad_Test' diff --git a/tests/ut/python/adv_robustness/detectors/__init__.py b/tests/ut/python/adv_robustness/detectors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2cecd48e7942c8da6326a8de0d2627d68eea2e79 --- /dev/null +++ b/tests/ut/python/adv_robustness/detectors/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for detection algorithms of white-box attacks. +""" diff --git a/tests/ut/python/adv_robustness/detectors/black/__init__.py b/tests/ut/python/adv_robustness/detectors/black/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..443a34a4eed4bf773776bca3ee3abd0749eed672 --- /dev/null +++ b/tests/ut/python/adv_robustness/detectors/black/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for detection algorithms of black-box attacks. +""" diff --git a/tests/ut/python/detectors/black/test_similarity_detector.py b/tests/ut/python/adv_robustness/detectors/black/test_similarity_detector.py similarity index 100% rename from tests/ut/python/detectors/black/test_similarity_detector.py rename to tests/ut/python/adv_robustness/detectors/black/test_similarity_detector.py diff --git a/tests/ut/python/detectors/test_ensemble_detector.py b/tests/ut/python/adv_robustness/detectors/test_ensemble_detector.py similarity index 100% rename from tests/ut/python/detectors/test_ensemble_detector.py rename to tests/ut/python/adv_robustness/detectors/test_ensemble_detector.py diff --git a/tests/ut/python/detectors/test_mag_net.py b/tests/ut/python/adv_robustness/detectors/test_mag_net.py similarity index 100% rename from tests/ut/python/detectors/test_mag_net.py rename to tests/ut/python/adv_robustness/detectors/test_mag_net.py diff --git a/tests/ut/python/detectors/test_region_based_detector.py b/tests/ut/python/adv_robustness/detectors/test_region_based_detector.py similarity index 100% rename from tests/ut/python/detectors/test_region_based_detector.py rename to tests/ut/python/adv_robustness/detectors/test_region_based_detector.py diff --git a/tests/ut/python/detectors/test_spatial_smoothing.py b/tests/ut/python/adv_robustness/detectors/test_spatial_smoothing.py similarity index 100% rename from tests/ut/python/detectors/test_spatial_smoothing.py rename to tests/ut/python/adv_robustness/detectors/test_spatial_smoothing.py diff --git a/tests/ut/python/adv_robustness/evaluations/__init__.py b/tests/ut/python/adv_robustness/evaluations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c8b4b13652f7de306dd44cbf50c0a8ecf8e59adf --- /dev/null +++ b/tests/ut/python/adv_robustness/evaluations/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for the evaluations of white-box attack and defense. +""" diff --git a/tests/ut/python/adv_robustness/evaluations/black/__init__.py b/tests/ut/python/adv_robustness/evaluations/black/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4f54ce8d9e69a14a966b51f4f7f19c446b8fd9b3 --- /dev/null +++ b/tests/ut/python/adv_robustness/evaluations/black/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for the evaluations of black-box attack and defense. +""" diff --git a/tests/ut/python/evaluations/black/test_black_defense_eval.py b/tests/ut/python/adv_robustness/evaluations/black/test_black_defense_eval.py similarity index 100% rename from tests/ut/python/evaluations/black/test_black_defense_eval.py rename to tests/ut/python/adv_robustness/evaluations/black/test_black_defense_eval.py diff --git a/tests/ut/python/evaluations/test_attack_eval.py b/tests/ut/python/adv_robustness/evaluations/test_attack_eval.py similarity index 100% rename from tests/ut/python/evaluations/test_attack_eval.py rename to tests/ut/python/adv_robustness/evaluations/test_attack_eval.py diff --git a/tests/ut/python/evaluations/test_defense_eval.py b/tests/ut/python/adv_robustness/evaluations/test_defense_eval.py similarity index 100% rename from tests/ut/python/evaluations/test_defense_eval.py rename to tests/ut/python/adv_robustness/evaluations/test_defense_eval.py diff --git a/tests/ut/python/evaluations/test_radar_metric.py b/tests/ut/python/adv_robustness/evaluations/test_radar_metric.py similarity index 98% rename from tests/ut/python/evaluations/test_radar_metric.py rename to tests/ut/python/adv_robustness/evaluations/test_radar_metric.py index f43db5ec89a2678afe3a259c6303f5286f15a95e..bdcb12058805da18f845bc81fb1f1667ad66216e 100644 --- a/tests/ut/python/evaluations/test_radar_metric.py +++ b/tests/ut/python/adv_robustness/evaluations/test_radar_metric.py @@ -16,6 +16,8 @@ Radar map test. """ import pytest from mindarmour.adv_robustness.evaluations import RadarMetric + + @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_ascend_training @@ -31,7 +33,7 @@ def test_radar_metric(): # create obj _ = RadarMetric(metrics_name, metrics_data, metrics_labels, title='', - scale='sparse') + scale='sparse') @pytest.mark.level0 diff --git a/tests/ut/python/test_data/test_images.npy b/tests/ut/python/dataset/test_images.npy similarity index 100% rename from tests/ut/python/test_data/test_images.npy rename to tests/ut/python/dataset/test_images.npy diff --git a/tests/ut/python/test_data/test_labels.npy b/tests/ut/python/dataset/test_labels.npy similarity index 100% rename from tests/ut/python/test_data/test_labels.npy rename to tests/ut/python/dataset/test_labels.npy diff --git a/tests/ut/python/test_data/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt b/tests/ut/python/dataset/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt similarity index 100% rename from tests/ut/python/test_data/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt rename to tests/ut/python/dataset/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt diff --git a/tests/ut/python/defenses/mock_net.py b/tests/ut/python/defenses/mock_net.py deleted file mode 100644 index db49ebc2fc66e73783ee6b91694935f27fda94a5..0000000000000000000000000000000000000000 --- a/tests/ut/python/defenses/mock_net.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2019 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -mocked model for UT of defense algorithms. -""" -import numpy as np - -from mindspore import nn -from mindspore import Tensor -from mindspore.nn import WithLossCell, TrainOneStepCell -from mindspore.nn.optim.momentum import Momentum -from mindspore import context -from mindspore.common.initializer import TruncatedNormal - -from mindarmour.adv_robustness.attacks import FastGradientSignMethod - - -def conv(in_channels, out_channels, kernel_size, stride=1, padding=0): - weight = weight_variable() - return nn.Conv2d(in_channels, out_channels, - kernel_size=kernel_size, stride=stride, padding=padding, - weight_init=weight, has_bias=False, pad_mode="valid") - - -def fc_with_initialize(input_channels, out_channels): - weight = weight_variable() - bias = weight_variable() - return nn.Dense(input_channels, out_channels, weight, bias) - - -def weight_variable(): - return TruncatedNormal(0.02) - - -class Net(nn.Cell): - """ - Lenet network - """ - def __init__(self): - super(Net, self).__init__() - self.conv1 = conv(1, 6, 5) - self.conv2 = conv(6, 16, 5) - self.fc1 = fc_with_initialize(16*5*5, 120) - self.fc2 = fc_with_initialize(120, 84) - self.fc3 = fc_with_initialize(84, 10) - self.relu = nn.ReLU() - self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) - self.flatten = nn.Flatten() - - def construct(self, x): - x = self.conv1(x) - x = self.relu(x) - x = self.max_pool2d(x) - x = self.conv2(x) - x = self.relu(x) - x = self.max_pool2d(x) - x = self.flatten(x) - x = self.fc1(x) - x = self.relu(x) - x = self.fc2(x) - x = self.relu(x) - x = self.fc3(x) - return x - - -if __name__ == '__main__': - num_classes = 10 - batch_size = 32 - - sparse = False - context.set_context(mode=context.GRAPH_MODE) - context.set_context(device_target='Ascend') - - # create test data - inputs_np = np.random.rand(batch_size, 1, 32, 32).astype(np.float32) - labels_np = np.random.randint(num_classes, size=batch_size).astype(np.int32) - if not sparse: - labels_np = np.eye(num_classes)[labels_np].astype(np.float32) - - net = Net() - - # test fgsm - attack = FastGradientSignMethod(net, eps=0.3) - attack.generate(inputs_np, labels_np) - - # test train ops - loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=sparse) - optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), - 0.01, 0.9) - loss_net = WithLossCell(net, loss_fn) - train_net = TrainOneStepCell(loss_net, optimizer) - train_net.set_train() - - train_net(Tensor(inputs_np), Tensor(labels_np)) diff --git a/tests/ut/python/fuzzing/__init__.py b/tests/ut/python/fuzzing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..41fabd7256c40302777d1c90599fd31c955f821e --- /dev/null +++ b/tests/ut/python/fuzzing/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for model fuzzing. +""" diff --git a/tests/ut/python/privacy/__init__.py b/tests/ut/python/privacy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d57ca645472b2213ef74e4d407fcef7a0bb07cfc --- /dev/null +++ b/tests/ut/python/privacy/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for differential-privacy training and +privacy breach estimation. +""" diff --git a/tests/ut/python/privacy/diff_privacy/__init__.py b/tests/ut/python/privacy/diff_privacy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..89d9484ed71c5e12aa5d3c7e9a1f7aea75efd724 --- /dev/null +++ b/tests/ut/python/privacy/diff_privacy/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for differential-privacy training and +privacy-budget estimation. +""" diff --git a/tests/ut/python/diff_privacy/test_mechanisms.py b/tests/ut/python/privacy/diff_privacy/test_mechanisms.py similarity index 100% rename from tests/ut/python/diff_privacy/test_mechanisms.py rename to tests/ut/python/privacy/diff_privacy/test_mechanisms.py diff --git a/tests/ut/python/diff_privacy/test_model_train.py b/tests/ut/python/privacy/diff_privacy/test_model_train.py similarity index 98% rename from tests/ut/python/diff_privacy/test_model_train.py rename to tests/ut/python/privacy/diff_privacy/test_model_train.py index 7608aaa0d4d9e82293b739a7db1924d82c4006f2..6211adef54646bb439c4ea95a3da270ca68d7e3f 100644 --- a/tests/ut/python/diff_privacy/test_model_train.py +++ b/tests/ut/python/privacy/diff_privacy/test_model_train.py @@ -26,7 +26,7 @@ from mindarmour.privacy.diff_privacy import NoiseMechanismsFactory from mindarmour.privacy.diff_privacy import ClipMechanismsFactory from mindarmour.privacy.diff_privacy import DPOptimizerClassFactory -from test_network import LeNet5 +from ut.python.utils.mock_net import Net def dataset_generator(batch_size, batches): @@ -48,7 +48,7 @@ def test_dp_model_with_pynative_mode(): context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend") norm_bound = 1.0 initial_noise_multiplier = 0.01 - network = LeNet5() + network = Net() batch_size = 32 batches = 128 epochs = 1 @@ -88,7 +88,7 @@ def test_dp_model_with_graph_mode(): context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") norm_bound = 1.0 initial_noise_multiplier = 0.01 - network = LeNet5() + network = Net() batch_size = 32 batches = 128 epochs = 1 @@ -126,7 +126,7 @@ def test_dp_model_with_graph_mode_ada_gaussian(): context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") norm_bound = 1.0 initial_noise_multiplier = 0.01 - network = LeNet5() + network = Net() batch_size = 32 batches = 128 epochs = 1 diff --git a/tests/ut/python/diff_privacy/test_monitor.py b/tests/ut/python/privacy/diff_privacy/test_monitor.py similarity index 98% rename from tests/ut/python/diff_privacy/test_monitor.py rename to tests/ut/python/privacy/diff_privacy/test_monitor.py index b0cc0cc9e6e5a58a5d8bfc77c08cc767aa4d67af..855796934fe8a75ef57aa9da2391b16175c6bff5 100644 --- a/tests/ut/python/diff_privacy/test_monitor.py +++ b/tests/ut/python/privacy/diff_privacy/test_monitor.py @@ -25,7 +25,7 @@ import mindspore.context as context from mindarmour.privacy.diff_privacy import PrivacyMonitorFactory from mindarmour.utils.logger import LogUtil -from test_network import LeNet5 +from ut.python.utils.mock_net import Net LOGGER = LogUtil.get_instance() TAG = 'DP-Monitor Test' @@ -57,7 +57,7 @@ def test_dp_monitor(): suggest_epoch = rdp.max_epoch_suggest() LOGGER.info(TAG, 'The recommended maximum training epochs is: %s', suggest_epoch) - network = LeNet5() + network = Net() net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) @@ -86,7 +86,7 @@ def test_dp_monitor_gpu(): suggest_epoch = rdp.max_epoch_suggest() LOGGER.info(TAG, 'The recommended maximum training epochs is: %s', suggest_epoch) - network = LeNet5() + network = Net() net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) @@ -115,7 +115,7 @@ def test_dp_monitor_cpu(): suggest_epoch = rdp.max_epoch_suggest() LOGGER.info(TAG, 'The recommended maximum training epochs is: %s', suggest_epoch) - network = LeNet5() + network = Net() net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) @@ -145,7 +145,7 @@ def test_dp_monitor_zcdp(): suggest_epoch = zcdp.max_epoch_suggest() LOGGER.info(TAG, 'The recommended maximum training epochs is: %s', suggest_epoch) - network = LeNet5() + network = Net() net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) @@ -174,7 +174,7 @@ def test_dp_monitor_zcdp_gpu(): suggest_epoch = zcdp.max_epoch_suggest() LOGGER.info(TAG, 'The recommended maximum training epochs is: %s', suggest_epoch) - network = LeNet5() + network = Net() net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) @@ -203,7 +203,7 @@ def test_dp_monitor_zcdp_cpu(): suggest_epoch = zcdp.max_epoch_suggest() LOGGER.info(TAG, 'The recommended maximum training epochs is: %s', suggest_epoch) - network = LeNet5() + network = Net() net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) diff --git a/tests/ut/python/diff_privacy/test_optimizer.py b/tests/ut/python/privacy/diff_privacy/test_optimizer.py similarity index 96% rename from tests/ut/python/diff_privacy/test_optimizer.py rename to tests/ut/python/privacy/diff_privacy/test_optimizer.py index 128ba0e2dd8337299c5152ddccc9243dfc218fb9..ca0831329d502fcccceaa6efe79afc2312d34124 100644 --- a/tests/ut/python/diff_privacy/test_optimizer.py +++ b/tests/ut/python/privacy/diff_privacy/test_optimizer.py @@ -19,7 +19,7 @@ from mindspore.train.model import Model from mindarmour.privacy.diff_privacy import DPOptimizerClassFactory -from test_network import LeNet5 +from ut.python.utils.mock_net import Net @pytest.mark.level0 @@ -29,7 +29,7 @@ from test_network import LeNet5 @pytest.mark.component_mindarmour def test_optimizer(): context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend") - network = LeNet5() + network = Net() lr = 0.01 momentum = 0.9 micro_batches = 2 @@ -47,7 +47,7 @@ def test_optimizer(): @pytest.mark.component_mindarmour def test_optimizer_gpu(): context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") - network = LeNet5() + network = Net() lr = 0.01 momentum = 0.9 micro_batches = 2 @@ -65,7 +65,7 @@ def test_optimizer_gpu(): @pytest.mark.component_mindarmour def test_optimizer_cpu(): context.set_context(mode=context.PYNATIVE_MODE, device_target="CPU") - network = LeNet5() + network = Net() lr = 0.01 momentum = 0.9 micro_batches = 2 diff --git a/tests/ut/python/privacy/evaluation/__init__.py b/tests/ut/python/privacy/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dc7bb709a42c0558cde74125f5c389c97cb34160 --- /dev/null +++ b/tests/ut/python/privacy/evaluation/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for privacy breach estimation. +""" diff --git a/tests/ut/python/diff_privacy/test_attacker.py b/tests/ut/python/privacy/evaluation/test_attacker.py similarity index 100% rename from tests/ut/python/diff_privacy/test_attacker.py rename to tests/ut/python/privacy/evaluation/test_attacker.py diff --git a/tests/ut/python/diff_privacy/test_membership_inference.py b/tests/ut/python/privacy/evaluation/test_membership_inference.py similarity index 96% rename from tests/ut/python/diff_privacy/test_membership_inference.py rename to tests/ut/python/privacy/evaluation/test_membership_inference.py index d4f925c4e08dc372115318bd3ce1a7c365d97478..1065f84eef599a2ed406c83af1175c6ac6d628a1 100644 --- a/tests/ut/python/diff_privacy/test_membership_inference.py +++ b/tests/ut/python/privacy/evaluation/test_membership_inference.py @@ -14,9 +14,6 @@ """ membership inference test """ -import os -import sys - import pytest import numpy as np @@ -28,12 +25,12 @@ import mindspore.context as context from mindarmour.privacy.evaluation import MembershipInference -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../")) -from defenses.mock_net import Net +from ut.python.utils.mock_net import Net context.set_context(mode=context.GRAPH_MODE) + def dataset_generator(batch_size, batches): """mock training data.""" data = np.random.randn(batches*batch_size, 1, 32, 32).astype( diff --git a/tests/ut/python/utils/__init__.py b/tests/ut/python/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ea65540163b90fb2e29ae2bf10dac35458b28479 --- /dev/null +++ b/tests/ut/python/utils/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes common methods called by unit tests. +""" diff --git a/tests/ut/python/diff_privacy/test_network.py b/tests/ut/python/utils/mock_net.py similarity index 93% rename from tests/ut/python/diff_privacy/test_network.py rename to tests/ut/python/utils/mock_net.py index ac5cf3bd095feeda765edffa888385c6e2a2368a..7b5b4bb9564cabb70728a266fa6ba5d75d2a4dff 100644 --- a/tests/ut/python/diff_privacy/test_network.py +++ b/tests/ut/python/utils/mock_net.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +""" +mocked model for UT of defense algorithms. +""" from mindspore import nn from mindspore.common.initializer import TruncatedNormal @@ -29,15 +32,15 @@ def fc_with_initialize(input_channels, out_channels): def weight_variable(): - return TruncatedNormal(0.05) + return TruncatedNormal(0.02) -class LeNet5(nn.Cell): +class Net(nn.Cell): """ Lenet network """ def __init__(self): - super(LeNet5, self).__init__() + super(Net, self).__init__() self.conv1 = conv(1, 6, 5) self.conv2 = conv(6, 16, 5) self.fc1 = fc_with_initialize(16*5*5, 120)