From 913e2fc98c6b6d86058eebc3c38a04abbd0df48b Mon Sep 17 00:00:00 2001 From: jin-xiulang Date: Wed, 9 Sep 2020 16:42:48 +0800 Subject: [PATCH] Adjust the structure of tests package. --- tests/ut/__init__.py | 16 +++ tests/ut/python/__init__.py | 16 +++ tests/ut/python/adv_robustness/__init__.py | 17 +++ .../python/adv_robustness/attacks/__init__.py | 16 +++ .../adv_robustness/attacks/black/__init__.py | 16 +++ .../attacks/black/test_genetic_attack.py | 0 .../attacks/black/test_hsja.py | 19 ++-- .../attacks/black/test_nes.py | 21 ++-- .../attacks/black/test_pointwise_attack.py | 14 +-- .../attacks/black/test_pso_attack.py | 0 .../black/test_salt_and_pepper_attack.py | 2 +- .../attacks/test_batch_generate_attack.py | 0 .../{ => adv_robustness}/attacks/test_cw.py | 0 .../attacks/test_deep_fool.py | 0 .../attacks/test_gradient_method.py | 0 .../attacks/test_iterative_gradient_method.py | 0 .../{ => adv_robustness}/attacks/test_jsma.py | 0 .../attacks/test_lbfgs.py | 13 +-- .../adv_robustness/defenses/__init__.py | 16 +++ .../{ => adv_robustness}/defenses/test_ad.py | 3 +- .../{ => adv_robustness}/defenses/test_ead.py | 3 +- .../{ => adv_robustness}/defenses/test_nad.py | 3 +- .../{ => adv_robustness}/defenses/test_pad.py | 3 +- .../adv_robustness/detectors/__init__.py | 16 +++ .../detectors/black/__init__.py | 16 +++ .../black/test_similarity_detector.py | 0 .../detectors/test_ensemble_detector.py | 0 .../detectors/test_mag_net.py | 0 .../detectors/test_region_based_detector.py | 0 .../detectors/test_spatial_smoothing.py | 0 .../adv_robustness/evaluations/__init__.py | 16 +++ .../evaluations/black/__init__.py | 16 +++ .../black/test_black_defense_eval.py | 0 .../evaluations/test_attack_eval.py | 0 .../evaluations/test_defense_eval.py | 0 .../evaluations/test_radar_metric.py | 4 +- .../{test_data => dataset}/test_images.npy | Bin .../{test_data => dataset}/test_labels.npy | Bin .../checkpoint_lenet-10_1875.ckpt | Bin tests/ut/python/defenses/mock_net.py | 105 ------------------ tests/ut/python/fuzzing/__init__.py | 16 +++ tests/ut/python/privacy/__init__.py | 17 +++ .../python/privacy/diff_privacy/__init__.py | 17 +++ .../diff_privacy/test_mechanisms.py | 0 .../diff_privacy/test_model_train.py | 8 +- .../diff_privacy/test_monitor.py | 14 +-- .../diff_privacy/test_optimizer.py | 8 +- .../ut/python/privacy/evaluation/__init__.py | 16 +++ .../evaluation}/test_attacker.py | 0 .../evaluation}/test_membership_inference.py | 7 +- tests/ut/python/utils/__init__.py | 16 +++ .../test_network.py => utils/mock_net.py} | 9 +- 52 files changed, 308 insertions(+), 171 deletions(-) create mode 100644 tests/ut/__init__.py create mode 100644 tests/ut/python/__init__.py create mode 100644 tests/ut/python/adv_robustness/__init__.py create mode 100644 tests/ut/python/adv_robustness/attacks/__init__.py create mode 100644 tests/ut/python/adv_robustness/attacks/black/__init__.py rename tests/ut/python/{ => adv_robustness}/attacks/black/test_genetic_attack.py (100%) rename tests/ut/python/{ => adv_robustness}/attacks/black/test_hsja.py (90%) rename tests/ut/python/{ => adv_robustness}/attacks/black/test_nes.py (92%) rename tests/ut/python/{ => adv_robustness}/attacks/black/test_pointwise_attack.py (86%) rename tests/ut/python/{ => adv_robustness}/attacks/black/test_pso_attack.py (100%) rename tests/ut/python/{ => adv_robustness}/attacks/black/test_salt_and_pepper_attack.py (98%) rename tests/ut/python/{ => adv_robustness}/attacks/test_batch_generate_attack.py (100%) rename tests/ut/python/{ => adv_robustness}/attacks/test_cw.py (100%) rename tests/ut/python/{ => adv_robustness}/attacks/test_deep_fool.py (100%) rename tests/ut/python/{ => adv_robustness}/attacks/test_gradient_method.py (100%) rename tests/ut/python/{ => adv_robustness}/attacks/test_iterative_gradient_method.py (100%) rename tests/ut/python/{ => adv_robustness}/attacks/test_jsma.py (100%) rename tests/ut/python/{ => adv_robustness}/attacks/test_lbfgs.py (82%) create mode 100644 tests/ut/python/adv_robustness/defenses/__init__.py rename tests/ut/python/{ => adv_robustness}/defenses/test_ad.py (98%) rename tests/ut/python/{ => adv_robustness}/defenses/test_ead.py (98%) rename tests/ut/python/{ => adv_robustness}/defenses/test_nad.py (97%) rename tests/ut/python/{ => adv_robustness}/defenses/test_pad.py (98%) create mode 100644 tests/ut/python/adv_robustness/detectors/__init__.py create mode 100644 tests/ut/python/adv_robustness/detectors/black/__init__.py rename tests/ut/python/{ => adv_robustness}/detectors/black/test_similarity_detector.py (100%) rename tests/ut/python/{ => adv_robustness}/detectors/test_ensemble_detector.py (100%) rename tests/ut/python/{ => adv_robustness}/detectors/test_mag_net.py (100%) rename tests/ut/python/{ => adv_robustness}/detectors/test_region_based_detector.py (100%) rename tests/ut/python/{ => adv_robustness}/detectors/test_spatial_smoothing.py (100%) create mode 100644 tests/ut/python/adv_robustness/evaluations/__init__.py create mode 100644 tests/ut/python/adv_robustness/evaluations/black/__init__.py rename tests/ut/python/{ => adv_robustness}/evaluations/black/test_black_defense_eval.py (100%) rename tests/ut/python/{ => adv_robustness}/evaluations/test_attack_eval.py (100%) rename tests/ut/python/{ => adv_robustness}/evaluations/test_defense_eval.py (100%) rename tests/ut/python/{ => adv_robustness}/evaluations/test_radar_metric.py (98%) rename tests/ut/python/{test_data => dataset}/test_images.npy (100%) rename tests/ut/python/{test_data => dataset}/test_labels.npy (100%) rename tests/ut/python/{test_data => dataset}/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt (100%) delete mode 100644 tests/ut/python/defenses/mock_net.py create mode 100644 tests/ut/python/fuzzing/__init__.py create mode 100644 tests/ut/python/privacy/__init__.py create mode 100644 tests/ut/python/privacy/diff_privacy/__init__.py rename tests/ut/python/{ => privacy}/diff_privacy/test_mechanisms.py (100%) rename tests/ut/python/{ => privacy}/diff_privacy/test_model_train.py (98%) rename tests/ut/python/{ => privacy}/diff_privacy/test_monitor.py (98%) rename tests/ut/python/{ => privacy}/diff_privacy/test_optimizer.py (96%) create mode 100644 tests/ut/python/privacy/evaluation/__init__.py rename tests/ut/python/{diff_privacy => privacy/evaluation}/test_attacker.py (100%) rename tests/ut/python/{diff_privacy => privacy/evaluation}/test_membership_inference.py (96%) create mode 100644 tests/ut/python/utils/__init__.py rename tests/ut/python/{diff_privacy/test_network.py => utils/mock_net.py} (93%) diff --git a/tests/ut/__init__.py b/tests/ut/__init__.py new file mode 100644 index 0000000..1fec660 --- /dev/null +++ b/tests/ut/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for developed features of MindArmour. +""" diff --git a/tests/ut/python/__init__.py b/tests/ut/python/__init__.py new file mode 100644 index 0000000..190835e --- /dev/null +++ b/tests/ut/python/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes python unit tests for developed features of MindArmour. +""" diff --git a/tests/ut/python/adv_robustness/__init__.py b/tests/ut/python/adv_robustness/__init__.py new file mode 100644 index 0000000..02fa417 --- /dev/null +++ b/tests/ut/python/adv_robustness/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for model attacks, model defenses and +their evaluation functions. +""" diff --git a/tests/ut/python/adv_robustness/attacks/__init__.py b/tests/ut/python/adv_robustness/attacks/__init__.py new file mode 100644 index 0000000..7c32a30 --- /dev/null +++ b/tests/ut/python/adv_robustness/attacks/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for white-box attack algorithms. +""" diff --git a/tests/ut/python/adv_robustness/attacks/black/__init__.py b/tests/ut/python/adv_robustness/attacks/black/__init__.py new file mode 100644 index 0000000..a0256d6 --- /dev/null +++ b/tests/ut/python/adv_robustness/attacks/black/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for black-box attack algorithms. +""" diff --git a/tests/ut/python/attacks/black/test_genetic_attack.py b/tests/ut/python/adv_robustness/attacks/black/test_genetic_attack.py similarity index 100% rename from tests/ut/python/attacks/black/test_genetic_attack.py rename to tests/ut/python/adv_robustness/attacks/black/test_genetic_attack.py diff --git a/tests/ut/python/attacks/black/test_hsja.py b/tests/ut/python/adv_robustness/attacks/black/test_hsja.py similarity index 90% rename from tests/ut/python/attacks/black/test_hsja.py rename to tests/ut/python/adv_robustness/attacks/black/test_hsja.py index b19e203..d4ddaee 100644 --- a/tests/ut/python/attacks/black/test_hsja.py +++ b/tests/ut/python/adv_robustness/attacks/black/test_hsja.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import sys import numpy as np import pytest @@ -24,9 +23,7 @@ from mindarmour import BlackModel from mindarmour.adv_robustness.attacks import HopSkipJumpAttack from mindarmour.utils.logger import LogUtil -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), - "../../../../../")) -from example.mnist_demo.lenet5_net import LeNet5 +from ut.python.utils.mock_net import Net context.set_context(mode=context.GRAPH_MODE) context.set_context(device_target="Ascend") @@ -64,25 +61,27 @@ def random_target_labels(true_labels): def create_target_images(dataset, data_labels, target_labels): res = [] for label in target_labels: - for i in range(len(data_labels)): - if data_labels[i] == label: + for i, data_label in enumerate(data_labels): + if data_label == label: res.append(dataset[i]) break return np.array(res) + # public variable def get_model(): # upload trained network current_dir = os.path.dirname(os.path.abspath(__file__)) ckpt_name = os.path.join(current_dir, - '../../test_data/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') - net = LeNet5() + '../../../dataset/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') + net = Net() load_dict = load_checkpoint(ckpt_name) load_param_into_net(net, load_dict) net.set_train(False) model = ModelToBeAttacked(net) return model + @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_ascend_training @@ -97,9 +96,9 @@ def test_hsja_mnist_attack(): # get test data test_images_set = np.load(os.path.join(current_dir, - '../../test_data/test_images.npy')) + '../../../dataset/test_images.npy')) test_labels_set = np.load(os.path.join(current_dir, - '../../test_data/test_labels.npy')) + '../../../dataset/test_labels.npy')) # prediction accuracy before attack model = get_model() batch_num = 1 # the number of batches of attacking samples diff --git a/tests/ut/python/attacks/black/test_nes.py b/tests/ut/python/adv_robustness/attacks/black/test_nes.py similarity index 92% rename from tests/ut/python/attacks/black/test_nes.py rename to tests/ut/python/adv_robustness/attacks/black/test_nes.py index 6a84920..d9fa516 100644 --- a/tests/ut/python/attacks/black/test_nes.py +++ b/tests/ut/python/adv_robustness/attacks/black/test_nes.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import sys import numpy as np import pytest @@ -24,9 +23,7 @@ from mindarmour import BlackModel from mindarmour.adv_robustness.attacks import NES from mindarmour.utils.logger import LogUtil -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), - "../../../../../")) -from example.mnist_demo.lenet5_net import LeNet5 +from ut.python.utils.mock_net import Net context.set_context(mode=context.GRAPH_MODE) context.set_context(device_target="Ascend") @@ -73,31 +70,35 @@ def _pseudorandom_target(index, total_indices, true_class): def create_target_images(dataset, data_labels, target_labels): res = [] for label in target_labels: - for i in range(len(data_labels)): - if data_labels[i] == label: + for i, data_label in enumerate(data_labels): + if data_label == label: res.append(dataset[i]) break return np.array(res) + def get_model(current_dir): ckpt_name = os.path.join(current_dir, - '../../test_data/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') - net = LeNet5() + '../../../dataset/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') + net = Net() load_dict = load_checkpoint(ckpt_name) load_param_into_net(net, load_dict) net.set_train(False) model = ModelToBeAttacked(net) return model + def get_dataset(current_dir): # upload trained network # get test data test_images = np.load(os.path.join(current_dir, - '../../test_data/test_images.npy')) + '../../../dataset/test_images.npy')) test_labels = np.load(os.path.join(current_dir, - '../../test_data/test_labels.npy')) + '../../../dataset/test_labels.npy')) return test_images, test_labels + + def nes_mnist_attack(scene, top_k): """ hsja-Attack test diff --git a/tests/ut/python/attacks/black/test_pointwise_attack.py b/tests/ut/python/adv_robustness/attacks/black/test_pointwise_attack.py similarity index 86% rename from tests/ut/python/attacks/black/test_pointwise_attack.py rename to tests/ut/python/adv_robustness/attacks/black/test_pointwise_attack.py index 28c2c06..6428cdb 100644 --- a/tests/ut/python/attacks/black/test_pointwise_attack.py +++ b/tests/ut/python/adv_robustness/attacks/black/test_pointwise_attack.py @@ -15,7 +15,6 @@ PointWise Attack test """ import os -import sys import numpy as np import pytest @@ -27,10 +26,7 @@ from mindarmour import BlackModel from mindarmour.adv_robustness.attacks import PointWiseAttack from mindarmour.utils.logger import LogUtil -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), - "../../../../../")) -from example.mnist_demo.lenet5_net import LeNet5 - +from ut.python.utils.mock_net import Net context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") @@ -65,16 +61,16 @@ def test_pointwise_attack_method(): # upload trained network current_dir = os.path.dirname(os.path.abspath(__file__)) ckpt_name = os.path.join(current_dir, - '../../test_data/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') - net = LeNet5() + '../../../dataset/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') + net = Net() load_dict = load_checkpoint(ckpt_name) load_param_into_net(net, load_dict) # get one mnist image input_np = np.load(os.path.join(current_dir, - '../../test_data/test_images.npy'))[:3] + '../../../dataset/test_images.npy'))[:3] labels = np.load(os.path.join(current_dir, - '../../test_data/test_labels.npy'))[:3] + '../../../dataset/test_labels.npy'))[:3] model = ModelToBeAttacked(net) pre_label = np.argmax(model.predict(input_np), axis=1) LOGGER.info(TAG, 'original sample predict labels are :{}'.format(pre_label)) diff --git a/tests/ut/python/attacks/black/test_pso_attack.py b/tests/ut/python/adv_robustness/attacks/black/test_pso_attack.py similarity index 100% rename from tests/ut/python/attacks/black/test_pso_attack.py rename to tests/ut/python/adv_robustness/attacks/black/test_pso_attack.py diff --git a/tests/ut/python/attacks/black/test_salt_and_pepper_attack.py b/tests/ut/python/adv_robustness/attacks/black/test_salt_and_pepper_attack.py similarity index 98% rename from tests/ut/python/attacks/black/test_salt_and_pepper_attack.py rename to tests/ut/python/adv_robustness/attacks/black/test_salt_and_pepper_attack.py index ad08b1b..28b4743 100644 --- a/tests/ut/python/attacks/black/test_salt_and_pepper_attack.py +++ b/tests/ut/python/adv_robustness/attacks/black/test_salt_and_pepper_attack.py @@ -90,7 +90,7 @@ def test_salt_and_pepper_attack_method(): labels = labels.astype(np.float32) attack = SaltAndPepperNoiseAttack(model, sparse=False) - is_adv, adv_data, query_times = attack.generate(inputs, labels) + _, adv_data, _ = attack.generate(inputs, labels) assert np.any(adv_data[0] != inputs[0]), 'Salt and pepper attack method: ' \ 'generate value must not be equal' \ ' to original value.' diff --git a/tests/ut/python/attacks/test_batch_generate_attack.py b/tests/ut/python/adv_robustness/attacks/test_batch_generate_attack.py similarity index 100% rename from tests/ut/python/attacks/test_batch_generate_attack.py rename to tests/ut/python/adv_robustness/attacks/test_batch_generate_attack.py diff --git a/tests/ut/python/attacks/test_cw.py b/tests/ut/python/adv_robustness/attacks/test_cw.py similarity index 100% rename from tests/ut/python/attacks/test_cw.py rename to tests/ut/python/adv_robustness/attacks/test_cw.py diff --git a/tests/ut/python/attacks/test_deep_fool.py b/tests/ut/python/adv_robustness/attacks/test_deep_fool.py similarity index 100% rename from tests/ut/python/attacks/test_deep_fool.py rename to tests/ut/python/adv_robustness/attacks/test_deep_fool.py diff --git a/tests/ut/python/attacks/test_gradient_method.py b/tests/ut/python/adv_robustness/attacks/test_gradient_method.py similarity index 100% rename from tests/ut/python/attacks/test_gradient_method.py rename to tests/ut/python/adv_robustness/attacks/test_gradient_method.py diff --git a/tests/ut/python/attacks/test_iterative_gradient_method.py b/tests/ut/python/adv_robustness/attacks/test_iterative_gradient_method.py similarity index 100% rename from tests/ut/python/attacks/test_iterative_gradient_method.py rename to tests/ut/python/adv_robustness/attacks/test_iterative_gradient_method.py diff --git a/tests/ut/python/attacks/test_jsma.py b/tests/ut/python/adv_robustness/attacks/test_jsma.py similarity index 100% rename from tests/ut/python/attacks/test_jsma.py rename to tests/ut/python/adv_robustness/attacks/test_jsma.py diff --git a/tests/ut/python/attacks/test_lbfgs.py b/tests/ut/python/adv_robustness/attacks/test_lbfgs.py similarity index 82% rename from tests/ut/python/attacks/test_lbfgs.py rename to tests/ut/python/adv_robustness/attacks/test_lbfgs.py index 097df4e..46e7f05 100644 --- a/tests/ut/python/attacks/test_lbfgs.py +++ b/tests/ut/python/adv_robustness/attacks/test_lbfgs.py @@ -15,7 +15,6 @@ LBFGS-Attack test. """ import os -import sys import numpy as np import pytest @@ -25,9 +24,7 @@ from mindspore.train.serialization import load_checkpoint, load_param_into_net from mindarmour.adv_robustness.attacks import LBFGS from mindarmour.utils.logger import LogUtil -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), - "../../../../")) -from example.mnist_demo.lenet5_net import LeNet5 +from ut.python.utils.mock_net import Net context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") @@ -50,16 +47,16 @@ def test_lbfgs_attack(): # upload trained network current_dir = os.path.dirname(os.path.abspath(__file__)) ckpt_name = os.path.join(current_dir, - '../test_data/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') - net = LeNet5() + '../../dataset/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt') + net = Net() load_dict = load_checkpoint(ckpt_name) load_param_into_net(net, load_dict) # get one mnist image input_np = np.load(os.path.join(current_dir, - '../test_data/test_images.npy'))[:1] + '../../dataset/test_images.npy'))[:1] label_np = np.load(os.path.join(current_dir, - '../test_data/test_labels.npy'))[:1] + '../../dataset/test_labels.npy'))[:1] LOGGER.debug(TAG, 'true label is :{}'.format(label_np[0])) classes = 10 target_np = np.random.randint(0, classes, 1) diff --git a/tests/ut/python/adv_robustness/defenses/__init__.py b/tests/ut/python/adv_robustness/defenses/__init__.py new file mode 100644 index 0000000..93f240d --- /dev/null +++ b/tests/ut/python/adv_robustness/defenses/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for defense algorithms. +""" diff --git a/tests/ut/python/defenses/test_ad.py b/tests/ut/python/adv_robustness/defenses/test_ad.py similarity index 98% rename from tests/ut/python/defenses/test_ad.py rename to tests/ut/python/adv_robustness/defenses/test_ad.py index d13b809..2c0470d 100644 --- a/tests/ut/python/defenses/test_ad.py +++ b/tests/ut/python/adv_robustness/defenses/test_ad.py @@ -23,10 +23,11 @@ from mindspore import context from mindspore import nn from mindspore.nn.optim.momentum import Momentum -from mock_net import Net from mindarmour.adv_robustness.defenses import AdversarialDefense from mindarmour.utils.logger import LogUtil +from ut.python.utils.mock_net import Net + LOGGER = LogUtil.get_instance() TAG = 'Ad_Test' diff --git a/tests/ut/python/defenses/test_ead.py b/tests/ut/python/adv_robustness/defenses/test_ead.py similarity index 98% rename from tests/ut/python/defenses/test_ead.py rename to tests/ut/python/adv_robustness/defenses/test_ead.py index ecf9dfa..eb057e7 100644 --- a/tests/ut/python/defenses/test_ead.py +++ b/tests/ut/python/adv_robustness/defenses/test_ead.py @@ -22,13 +22,14 @@ from mindspore import context from mindspore import nn from mindspore.nn.optim.momentum import Momentum -from mock_net import Net from mindarmour.adv_robustness.attacks import FastGradientSignMethod from mindarmour.adv_robustness.attacks import \ ProjectedGradientDescent from mindarmour.adv_robustness.defenses import EnsembleAdversarialDefense from mindarmour.utils.logger import LogUtil +from ut.python.utils.mock_net import Net + LOGGER = LogUtil.get_instance() TAG = 'Ead_Test' diff --git a/tests/ut/python/defenses/test_nad.py b/tests/ut/python/adv_robustness/defenses/test_nad.py similarity index 97% rename from tests/ut/python/defenses/test_nad.py rename to tests/ut/python/adv_robustness/defenses/test_nad.py index c61fef8..8c0e5e0 100644 --- a/tests/ut/python/defenses/test_nad.py +++ b/tests/ut/python/adv_robustness/defenses/test_nad.py @@ -22,10 +22,11 @@ from mindspore import context from mindspore import nn from mindspore.nn.optim.momentum import Momentum -from mock_net import Net from mindarmour.adv_robustness.defenses import NaturalAdversarialDefense from mindarmour.utils.logger import LogUtil +from ut.python.utils.mock_net import Net + LOGGER = LogUtil.get_instance() TAG = 'Nad_Test' diff --git a/tests/ut/python/defenses/test_pad.py b/tests/ut/python/adv_robustness/defenses/test_pad.py similarity index 98% rename from tests/ut/python/defenses/test_pad.py rename to tests/ut/python/adv_robustness/defenses/test_pad.py index c8ea77d..fb4ed3a 100644 --- a/tests/ut/python/defenses/test_pad.py +++ b/tests/ut/python/adv_robustness/defenses/test_pad.py @@ -22,10 +22,11 @@ from mindspore import context from mindspore import nn from mindspore.nn.optim.momentum import Momentum -from mock_net import Net from mindarmour.adv_robustness.defenses import ProjectedAdversarialDefense from mindarmour.utils.logger import LogUtil +from ut.python.utils.mock_net import Net + LOGGER = LogUtil.get_instance() TAG = 'Pad_Test' diff --git a/tests/ut/python/adv_robustness/detectors/__init__.py b/tests/ut/python/adv_robustness/detectors/__init__.py new file mode 100644 index 0000000..2cecd48 --- /dev/null +++ b/tests/ut/python/adv_robustness/detectors/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for detection algorithms of white-box attacks. +""" diff --git a/tests/ut/python/adv_robustness/detectors/black/__init__.py b/tests/ut/python/adv_robustness/detectors/black/__init__.py new file mode 100644 index 0000000..443a34a --- /dev/null +++ b/tests/ut/python/adv_robustness/detectors/black/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for detection algorithms of black-box attacks. +""" diff --git a/tests/ut/python/detectors/black/test_similarity_detector.py b/tests/ut/python/adv_robustness/detectors/black/test_similarity_detector.py similarity index 100% rename from tests/ut/python/detectors/black/test_similarity_detector.py rename to tests/ut/python/adv_robustness/detectors/black/test_similarity_detector.py diff --git a/tests/ut/python/detectors/test_ensemble_detector.py b/tests/ut/python/adv_robustness/detectors/test_ensemble_detector.py similarity index 100% rename from tests/ut/python/detectors/test_ensemble_detector.py rename to tests/ut/python/adv_robustness/detectors/test_ensemble_detector.py diff --git a/tests/ut/python/detectors/test_mag_net.py b/tests/ut/python/adv_robustness/detectors/test_mag_net.py similarity index 100% rename from tests/ut/python/detectors/test_mag_net.py rename to tests/ut/python/adv_robustness/detectors/test_mag_net.py diff --git a/tests/ut/python/detectors/test_region_based_detector.py b/tests/ut/python/adv_robustness/detectors/test_region_based_detector.py similarity index 100% rename from tests/ut/python/detectors/test_region_based_detector.py rename to tests/ut/python/adv_robustness/detectors/test_region_based_detector.py diff --git a/tests/ut/python/detectors/test_spatial_smoothing.py b/tests/ut/python/adv_robustness/detectors/test_spatial_smoothing.py similarity index 100% rename from tests/ut/python/detectors/test_spatial_smoothing.py rename to tests/ut/python/adv_robustness/detectors/test_spatial_smoothing.py diff --git a/tests/ut/python/adv_robustness/evaluations/__init__.py b/tests/ut/python/adv_robustness/evaluations/__init__.py new file mode 100644 index 0000000..c8b4b13 --- /dev/null +++ b/tests/ut/python/adv_robustness/evaluations/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for the evaluations of white-box attack and defense. +""" diff --git a/tests/ut/python/adv_robustness/evaluations/black/__init__.py b/tests/ut/python/adv_robustness/evaluations/black/__init__.py new file mode 100644 index 0000000..4f54ce8 --- /dev/null +++ b/tests/ut/python/adv_robustness/evaluations/black/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for the evaluations of black-box attack and defense. +""" diff --git a/tests/ut/python/evaluations/black/test_black_defense_eval.py b/tests/ut/python/adv_robustness/evaluations/black/test_black_defense_eval.py similarity index 100% rename from tests/ut/python/evaluations/black/test_black_defense_eval.py rename to tests/ut/python/adv_robustness/evaluations/black/test_black_defense_eval.py diff --git a/tests/ut/python/evaluations/test_attack_eval.py b/tests/ut/python/adv_robustness/evaluations/test_attack_eval.py similarity index 100% rename from tests/ut/python/evaluations/test_attack_eval.py rename to tests/ut/python/adv_robustness/evaluations/test_attack_eval.py diff --git a/tests/ut/python/evaluations/test_defense_eval.py b/tests/ut/python/adv_robustness/evaluations/test_defense_eval.py similarity index 100% rename from tests/ut/python/evaluations/test_defense_eval.py rename to tests/ut/python/adv_robustness/evaluations/test_defense_eval.py diff --git a/tests/ut/python/evaluations/test_radar_metric.py b/tests/ut/python/adv_robustness/evaluations/test_radar_metric.py similarity index 98% rename from tests/ut/python/evaluations/test_radar_metric.py rename to tests/ut/python/adv_robustness/evaluations/test_radar_metric.py index f43db5e..bdcb120 100644 --- a/tests/ut/python/evaluations/test_radar_metric.py +++ b/tests/ut/python/adv_robustness/evaluations/test_radar_metric.py @@ -16,6 +16,8 @@ Radar map test. """ import pytest from mindarmour.adv_robustness.evaluations import RadarMetric + + @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_ascend_training @@ -31,7 +33,7 @@ def test_radar_metric(): # create obj _ = RadarMetric(metrics_name, metrics_data, metrics_labels, title='', - scale='sparse') + scale='sparse') @pytest.mark.level0 diff --git a/tests/ut/python/test_data/test_images.npy b/tests/ut/python/dataset/test_images.npy similarity index 100% rename from tests/ut/python/test_data/test_images.npy rename to tests/ut/python/dataset/test_images.npy diff --git a/tests/ut/python/test_data/test_labels.npy b/tests/ut/python/dataset/test_labels.npy similarity index 100% rename from tests/ut/python/test_data/test_labels.npy rename to tests/ut/python/dataset/test_labels.npy diff --git a/tests/ut/python/test_data/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt b/tests/ut/python/dataset/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt similarity index 100% rename from tests/ut/python/test_data/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt rename to tests/ut/python/dataset/trained_ckpt_file/checkpoint_lenet-10_1875.ckpt diff --git a/tests/ut/python/defenses/mock_net.py b/tests/ut/python/defenses/mock_net.py deleted file mode 100644 index db49ebc..0000000 --- a/tests/ut/python/defenses/mock_net.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2019 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -mocked model for UT of defense algorithms. -""" -import numpy as np - -from mindspore import nn -from mindspore import Tensor -from mindspore.nn import WithLossCell, TrainOneStepCell -from mindspore.nn.optim.momentum import Momentum -from mindspore import context -from mindspore.common.initializer import TruncatedNormal - -from mindarmour.adv_robustness.attacks import FastGradientSignMethod - - -def conv(in_channels, out_channels, kernel_size, stride=1, padding=0): - weight = weight_variable() - return nn.Conv2d(in_channels, out_channels, - kernel_size=kernel_size, stride=stride, padding=padding, - weight_init=weight, has_bias=False, pad_mode="valid") - - -def fc_with_initialize(input_channels, out_channels): - weight = weight_variable() - bias = weight_variable() - return nn.Dense(input_channels, out_channels, weight, bias) - - -def weight_variable(): - return TruncatedNormal(0.02) - - -class Net(nn.Cell): - """ - Lenet network - """ - def __init__(self): - super(Net, self).__init__() - self.conv1 = conv(1, 6, 5) - self.conv2 = conv(6, 16, 5) - self.fc1 = fc_with_initialize(16*5*5, 120) - self.fc2 = fc_with_initialize(120, 84) - self.fc3 = fc_with_initialize(84, 10) - self.relu = nn.ReLU() - self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) - self.flatten = nn.Flatten() - - def construct(self, x): - x = self.conv1(x) - x = self.relu(x) - x = self.max_pool2d(x) - x = self.conv2(x) - x = self.relu(x) - x = self.max_pool2d(x) - x = self.flatten(x) - x = self.fc1(x) - x = self.relu(x) - x = self.fc2(x) - x = self.relu(x) - x = self.fc3(x) - return x - - -if __name__ == '__main__': - num_classes = 10 - batch_size = 32 - - sparse = False - context.set_context(mode=context.GRAPH_MODE) - context.set_context(device_target='Ascend') - - # create test data - inputs_np = np.random.rand(batch_size, 1, 32, 32).astype(np.float32) - labels_np = np.random.randint(num_classes, size=batch_size).astype(np.int32) - if not sparse: - labels_np = np.eye(num_classes)[labels_np].astype(np.float32) - - net = Net() - - # test fgsm - attack = FastGradientSignMethod(net, eps=0.3) - attack.generate(inputs_np, labels_np) - - # test train ops - loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=sparse) - optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), - 0.01, 0.9) - loss_net = WithLossCell(net, loss_fn) - train_net = TrainOneStepCell(loss_net, optimizer) - train_net.set_train() - - train_net(Tensor(inputs_np), Tensor(labels_np)) diff --git a/tests/ut/python/fuzzing/__init__.py b/tests/ut/python/fuzzing/__init__.py new file mode 100644 index 0000000..41fabd7 --- /dev/null +++ b/tests/ut/python/fuzzing/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for model fuzzing. +""" diff --git a/tests/ut/python/privacy/__init__.py b/tests/ut/python/privacy/__init__.py new file mode 100644 index 0000000..d57ca64 --- /dev/null +++ b/tests/ut/python/privacy/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for differential-privacy training and +privacy breach estimation. +""" diff --git a/tests/ut/python/privacy/diff_privacy/__init__.py b/tests/ut/python/privacy/diff_privacy/__init__.py new file mode 100644 index 0000000..89d9484 --- /dev/null +++ b/tests/ut/python/privacy/diff_privacy/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for differential-privacy training and +privacy-budget estimation. +""" diff --git a/tests/ut/python/diff_privacy/test_mechanisms.py b/tests/ut/python/privacy/diff_privacy/test_mechanisms.py similarity index 100% rename from tests/ut/python/diff_privacy/test_mechanisms.py rename to tests/ut/python/privacy/diff_privacy/test_mechanisms.py diff --git a/tests/ut/python/diff_privacy/test_model_train.py b/tests/ut/python/privacy/diff_privacy/test_model_train.py similarity index 98% rename from tests/ut/python/diff_privacy/test_model_train.py rename to tests/ut/python/privacy/diff_privacy/test_model_train.py index 7608aaa..6211ade 100644 --- a/tests/ut/python/diff_privacy/test_model_train.py +++ b/tests/ut/python/privacy/diff_privacy/test_model_train.py @@ -26,7 +26,7 @@ from mindarmour.privacy.diff_privacy import NoiseMechanismsFactory from mindarmour.privacy.diff_privacy import ClipMechanismsFactory from mindarmour.privacy.diff_privacy import DPOptimizerClassFactory -from test_network import LeNet5 +from ut.python.utils.mock_net import Net def dataset_generator(batch_size, batches): @@ -48,7 +48,7 @@ def test_dp_model_with_pynative_mode(): context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend") norm_bound = 1.0 initial_noise_multiplier = 0.01 - network = LeNet5() + network = Net() batch_size = 32 batches = 128 epochs = 1 @@ -88,7 +88,7 @@ def test_dp_model_with_graph_mode(): context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") norm_bound = 1.0 initial_noise_multiplier = 0.01 - network = LeNet5() + network = Net() batch_size = 32 batches = 128 epochs = 1 @@ -126,7 +126,7 @@ def test_dp_model_with_graph_mode_ada_gaussian(): context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") norm_bound = 1.0 initial_noise_multiplier = 0.01 - network = LeNet5() + network = Net() batch_size = 32 batches = 128 epochs = 1 diff --git a/tests/ut/python/diff_privacy/test_monitor.py b/tests/ut/python/privacy/diff_privacy/test_monitor.py similarity index 98% rename from tests/ut/python/diff_privacy/test_monitor.py rename to tests/ut/python/privacy/diff_privacy/test_monitor.py index b0cc0cc..8557969 100644 --- a/tests/ut/python/diff_privacy/test_monitor.py +++ b/tests/ut/python/privacy/diff_privacy/test_monitor.py @@ -25,7 +25,7 @@ import mindspore.context as context from mindarmour.privacy.diff_privacy import PrivacyMonitorFactory from mindarmour.utils.logger import LogUtil -from test_network import LeNet5 +from ut.python.utils.mock_net import Net LOGGER = LogUtil.get_instance() TAG = 'DP-Monitor Test' @@ -57,7 +57,7 @@ def test_dp_monitor(): suggest_epoch = rdp.max_epoch_suggest() LOGGER.info(TAG, 'The recommended maximum training epochs is: %s', suggest_epoch) - network = LeNet5() + network = Net() net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) @@ -86,7 +86,7 @@ def test_dp_monitor_gpu(): suggest_epoch = rdp.max_epoch_suggest() LOGGER.info(TAG, 'The recommended maximum training epochs is: %s', suggest_epoch) - network = LeNet5() + network = Net() net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) @@ -115,7 +115,7 @@ def test_dp_monitor_cpu(): suggest_epoch = rdp.max_epoch_suggest() LOGGER.info(TAG, 'The recommended maximum training epochs is: %s', suggest_epoch) - network = LeNet5() + network = Net() net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) @@ -145,7 +145,7 @@ def test_dp_monitor_zcdp(): suggest_epoch = zcdp.max_epoch_suggest() LOGGER.info(TAG, 'The recommended maximum training epochs is: %s', suggest_epoch) - network = LeNet5() + network = Net() net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) @@ -174,7 +174,7 @@ def test_dp_monitor_zcdp_gpu(): suggest_epoch = zcdp.max_epoch_suggest() LOGGER.info(TAG, 'The recommended maximum training epochs is: %s', suggest_epoch) - network = LeNet5() + network = Net() net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) @@ -203,7 +203,7 @@ def test_dp_monitor_zcdp_cpu(): suggest_epoch = zcdp.max_epoch_suggest() LOGGER.info(TAG, 'The recommended maximum training epochs is: %s', suggest_epoch) - network = LeNet5() + network = Net() net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) diff --git a/tests/ut/python/diff_privacy/test_optimizer.py b/tests/ut/python/privacy/diff_privacy/test_optimizer.py similarity index 96% rename from tests/ut/python/diff_privacy/test_optimizer.py rename to tests/ut/python/privacy/diff_privacy/test_optimizer.py index 128ba0e..ca08313 100644 --- a/tests/ut/python/diff_privacy/test_optimizer.py +++ b/tests/ut/python/privacy/diff_privacy/test_optimizer.py @@ -19,7 +19,7 @@ from mindspore.train.model import Model from mindarmour.privacy.diff_privacy import DPOptimizerClassFactory -from test_network import LeNet5 +from ut.python.utils.mock_net import Net @pytest.mark.level0 @@ -29,7 +29,7 @@ from test_network import LeNet5 @pytest.mark.component_mindarmour def test_optimizer(): context.set_context(mode=context.PYNATIVE_MODE, device_target="Ascend") - network = LeNet5() + network = Net() lr = 0.01 momentum = 0.9 micro_batches = 2 @@ -47,7 +47,7 @@ def test_optimizer(): @pytest.mark.component_mindarmour def test_optimizer_gpu(): context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") - network = LeNet5() + network = Net() lr = 0.01 momentum = 0.9 micro_batches = 2 @@ -65,7 +65,7 @@ def test_optimizer_gpu(): @pytest.mark.component_mindarmour def test_optimizer_cpu(): context.set_context(mode=context.PYNATIVE_MODE, device_target="CPU") - network = LeNet5() + network = Net() lr = 0.01 momentum = 0.9 micro_batches = 2 diff --git a/tests/ut/python/privacy/evaluation/__init__.py b/tests/ut/python/privacy/evaluation/__init__.py new file mode 100644 index 0000000..dc7bb70 --- /dev/null +++ b/tests/ut/python/privacy/evaluation/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes unit tests for privacy breach estimation. +""" diff --git a/tests/ut/python/diff_privacy/test_attacker.py b/tests/ut/python/privacy/evaluation/test_attacker.py similarity index 100% rename from tests/ut/python/diff_privacy/test_attacker.py rename to tests/ut/python/privacy/evaluation/test_attacker.py diff --git a/tests/ut/python/diff_privacy/test_membership_inference.py b/tests/ut/python/privacy/evaluation/test_membership_inference.py similarity index 96% rename from tests/ut/python/diff_privacy/test_membership_inference.py rename to tests/ut/python/privacy/evaluation/test_membership_inference.py index d4f925c..1065f84 100644 --- a/tests/ut/python/diff_privacy/test_membership_inference.py +++ b/tests/ut/python/privacy/evaluation/test_membership_inference.py @@ -14,9 +14,6 @@ """ membership inference test """ -import os -import sys - import pytest import numpy as np @@ -28,12 +25,12 @@ import mindspore.context as context from mindarmour.privacy.evaluation import MembershipInference -sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../")) -from defenses.mock_net import Net +from ut.python.utils.mock_net import Net context.set_context(mode=context.GRAPH_MODE) + def dataset_generator(batch_size, batches): """mock training data.""" data = np.random.randn(batches*batch_size, 1, 32, 32).astype( diff --git a/tests/ut/python/utils/__init__.py b/tests/ut/python/utils/__init__.py new file mode 100644 index 0000000..ea65540 --- /dev/null +++ b/tests/ut/python/utils/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This package includes common methods called by unit tests. +""" diff --git a/tests/ut/python/diff_privacy/test_network.py b/tests/ut/python/utils/mock_net.py similarity index 93% rename from tests/ut/python/diff_privacy/test_network.py rename to tests/ut/python/utils/mock_net.py index ac5cf3b..7b5b4bb 100644 --- a/tests/ut/python/diff_privacy/test_network.py +++ b/tests/ut/python/utils/mock_net.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +""" +mocked model for UT of defense algorithms. +""" from mindspore import nn from mindspore.common.initializer import TruncatedNormal @@ -29,15 +32,15 @@ def fc_with_initialize(input_channels, out_channels): def weight_variable(): - return TruncatedNormal(0.05) + return TruncatedNormal(0.02) -class LeNet5(nn.Cell): +class Net(nn.Cell): """ Lenet network """ def __init__(self): - super(LeNet5, self).__init__() + super(Net, self).__init__() self.conv1 = conv(1, 6, 5) self.conv2 = conv(6, 16, 5) self.fc1 = fc_with_initialize(16*5*5, 120) -- GitLab