diff --git a/example/mnist_demo/lenet5_config.py b/example/mnist_demo/lenet5_config.py index 405b5ff5fa9d540248301d02ac5f262fa3e12f99..f1471b3b9d2f2910894f0cd48d6e880c40c9ded8 100644 --- a/example/mnist_demo/lenet5_config.py +++ b/example/mnist_demo/lenet5_config.py @@ -23,10 +23,10 @@ mnist_cfg = edict({ 'lr': 0.01, 'momentum': 0.9, 'epoch_size': 10, - 'batch_size': 32, + 'batch_size': 256, 'buffer_size': 1000, 'image_height': 32, 'image_width': 32, - 'save_checkpoint_steps': 1875, + 'save_checkpoint_steps': 234, 'keep_checkpoint_max': 10, }) diff --git a/example/mnist_demo/lenet5_dp_model_train.py b/example/mnist_demo/lenet5_dp_model_train.py index f01bfafe715125022162579d105872469f2fe437..a8cd58c36f407cdfdb6b9ace7fa36ac3e0f140c2 100644 --- a/example/mnist_demo/lenet5_dp_model_train.py +++ b/example/mnist_demo/lenet5_dp_model_train.py @@ -38,6 +38,7 @@ from lenet5_net import LeNet5 from lenet5_config import mnist_cfg as cfg LOGGER = LogUtil.get_instance() +LOGGER.set_level('INFO') TAG = 'Lenet5_train' @@ -92,11 +93,11 @@ if __name__ == "__main__": parser.add_argument('--data_path', type=str, default="./MNIST_unzip", help='path where the dataset is saved') parser.add_argument('--dataset_sink_mode', type=bool, default=False, help='dataset_sink_mode is False or True') - parser.add_argument('--micro_batches', type=int, default=None, + parser.add_argument('--micro_batches', type=int, default=32, help='optional, if use differential privacy, need to set micro_batches') - parser.add_argument('--l2_norm_bound', type=float, default=0.1, + parser.add_argument('--l2_norm_bound', type=float, default=1.0, help='optional, if use differential privacy, need to set l2_norm_bound') - parser.add_argument('--initial_noise_multiplier', type=float, default=0.001, + parser.add_argument('--initial_noise_multiplier', type=float, default=1.5, help='optional, if use differential privacy, need to set initial_noise_multiplier') args = parser.parse_args() @@ -120,13 +121,14 @@ if __name__ == "__main__": gaussian_mech.set_mechanisms('Gaussian', norm_bound=args.l2_norm_bound, initial_noise_multiplier=args.initial_noise_multiplier) - net_opt = gaussian_mech.create('SGD')(params=network.trainable_params(), - learning_rate=cfg.lr, - momentum=cfg.momentum) + net_opt = gaussian_mech.create('Momentum')(params=network.trainable_params(), + learning_rate=cfg.lr, + momentum=cfg.momentum) rdp_monitor = PrivacyMonitorFactory.create('rdp', num_samples=60000, batch_size=cfg.batch_size, - initial_noise_multiplier=args.initial_noise_multiplier, + initial_noise_multiplier=args.initial_noise_multiplier* + args.l2_norm_bound, per_print_times=10) model = DPModel(micro_batches=args.micro_batches, norm_clip=args.l2_norm_bound, @@ -141,7 +143,7 @@ if __name__ == "__main__": dataset_sink_mode=args.dataset_sink_mode) LOGGER.info(TAG, "============== Starting Testing ==============") - ckpt_file_name = 'trained_ckpt_file/checkpoint_lenet-10_1875.ckpt' + ckpt_file_name = 'trained_ckpt_file/checkpoint_lenet-10_234.ckpt' param_dict = load_checkpoint(ckpt_file_name) load_param_into_net(network, param_dict) ds_eval = generate_mnist_dataset(os.path.join(args.data_path, 'test'), batch_size=cfg.batch_size) diff --git a/example/mnist_demo/lenet5_net.py b/example/mnist_demo/lenet5_net.py index 7f5ead321cce8441bf204b5a1102f1d1712bcdbc..ac5cf3bd095feeda765edffa888385c6e2a2368a 100644 --- a/example/mnist_demo/lenet5_net.py +++ b/example/mnist_demo/lenet5_net.py @@ -29,7 +29,7 @@ def fc_with_initialize(input_channels, out_channels): def weight_variable(): - return TruncatedNormal(0.02) + return TruncatedNormal(0.05) class LeNet5(nn.Cell): diff --git a/mindarmour/diff_privacy/mechanisms/mechanisms.py b/mindarmour/diff_privacy/mechanisms/mechanisms.py index f0608c4ab0c38c2f2595d39ac082c3419b8a20ee..dd8f5da4ceef44e9884aa22a85f4b41a0886664a 100644 --- a/mindarmour/diff_privacy/mechanisms/mechanisms.py +++ b/mindarmour/diff_privacy/mechanisms/mechanisms.py @@ -72,24 +72,24 @@ class GaussianRandom(Mechanisms): Args: norm_bound(float): Clipping bound for the l2 norm of the gradients. - Default: 1.5. + Default: 1.0. initial_noise_multiplier(float): Ratio of the standard deviation of Gaussian noise divided by the norm_bound, which will be used to - calculate privacy spent. Default: 5.0. + calculate privacy spent. Default: 1.5. Returns: Tensor, generated noise. Examples: >>> shape = (3, 2, 4) - >>> norm_bound = 1.5 - >>> initial_noise_multiplier = 0.1 + >>> norm_bound = 1.0 + >>> initial_noise_multiplier = 1.5 >>> net = GaussianRandom(shape, norm_bound, initial_noise_multiplier) >>> res = net(shape) >>> print(res) """ - def __init__(self, norm_bound=1.5, initial_noise_multiplier=5.0): + def __init__(self, norm_bound=1.0, initial_noise_multiplier=1.5): super(GaussianRandom, self).__init__() self._norm_bound = check_value_positive('norm_bound', norm_bound) self._initial_noise_multiplier = check_value_positive('initial_noise_multiplier', diff --git a/mindarmour/diff_privacy/monitor/monitor.py b/mindarmour/diff_privacy/monitor/monitor.py index ca7bdf7564a6f9cf42db738916c1746116fe2835..d3fedc0577ffe1c02ddfaf6626f053090a54513f 100644 --- a/mindarmour/diff_privacy/monitor/monitor.py +++ b/mindarmour/diff_privacy/monitor/monitor.py @@ -70,11 +70,11 @@ class RDPMonitor(Callback): num_samples (int): The total number of samples in training data sets. batch_size (int): The number of samples in a batch while training. initial_noise_multiplier (Union[float, int]): The initial - multiplier of added noise. Default: 0.4. + multiplier of added noise. Default: 1.5. max_eps (Union[float, int, None]): The maximum acceptable epsilon - budget for DP training. Default: 3.0. + budget for DP training. Default: 10.0. target_delta (Union[float, int, None]): Target delta budget for DP - training. Default: 1e-5. + training. Default: 1e-3. max_delta (Union[float, int, None]): The maximum acceptable delta budget for DP training. Max_delta must be less than 1 and suggested to be less than 1e-3, otherwise overflow would be @@ -84,7 +84,7 @@ class RDPMonitor(Callback): orders (Union[None, list[int, float]]): Finite orders used for computing rdp, which must be greater than 1. noise_decay_mode (str): Decay mode of adding noise while training, - which can be 'no_decay', 'time' or 'step'. Default: 'step'. + which can be 'no_decay', 'Time' or 'Step'. Default: 'Time'. noise_decay_rate (Union[float, None]): Decay rate of noise while training. Default: 6e-4. per_print_times　(int): The interval steps of computing and printing @@ -92,7 +92,7 @@ class RDPMonitor(Callback): Examples: >>> rdp = PrivacyMonitorFactory.create(policy='rdp', - >>> num_samples=60000, batch_size=32) + >>> num_samples=60000, batch_size=256) >>> network = Net() >>> net_loss = nn.SoftmaxCrossEntropyWithLogits() >>> net_opt = nn.Momentum(network.trainable_params(), 0.01, 0.9) @@ -100,9 +100,9 @@ class RDPMonitor(Callback): >>> model.train(epochs, ds, callbacks=[rdp], dataset_sink_mode=False) """ - def __init__(self, num_samples, batch_size, initial_noise_multiplier=0.4, - max_eps=3.0, target_delta=1e-5, max_delta=None, - target_eps=None, orders=None, noise_decay_mode='step', + def __init__(self, num_samples, batch_size, initial_noise_multiplier=1.5, + max_eps=10.0, target_delta=1e-3, max_delta=None, + target_eps=None, orders=None, noise_decay_mode='Time', noise_decay_rate=6e-4, per_print_times=50): super(RDPMonitor, self).__init__() check_int_positive('num_samples', num_samples) @@ -132,8 +132,8 @@ class RDPMonitor(Callback): msg = 'orders must be greater than 1' LOGGER.error(TAG, msg) raise ValueError(msg) - if noise_decay_mode not in ('no_decay', 'step', 'time'): - msg = 'Noise decay mode must be in (no_decay, step, time)' + if noise_decay_mode not in ('no_decay', 'Step', 'Time'): + msg = "Noise decay mode must be in ('no_decay', 'Step', 'Time')" LOGGER.error(TAG, msg) raise ValueError(msg) if noise_decay_rate is not None: @@ -256,11 +256,11 @@ class RDPMonitor(Callback): LOGGER.error(TAG, msg) raise ValueError(msg) - if self._noise_decay_mode == 'time': + if self._noise_decay_mode == 'Time': noise_step = [self._initial_noise_multiplier / ( 1 + self._noise_decay_rate * step) for step in steps] - elif self._noise_decay_mode == 'step': + elif self._noise_decay_mode == 'Step': noise_step = [self._initial_noise_multiplier * ( 1 - self._noise_decay_rate) ** step for step in steps] self._rdp += sum( diff --git a/mindarmour/diff_privacy/optimizer/optimizer.py b/mindarmour/diff_privacy/optimizer/optimizer.py index a844e79d7e51d689f51eb2bdf72ff7d4050566a0..afedd2788c13729eb98fd0bb51005f441403b282 100644 --- a/mindarmour/diff_privacy/optimizer/optimizer.py +++ b/mindarmour/diff_privacy/optimizer/optimizer.py @@ -34,8 +34,8 @@ class DPOptimizerClassFactory: Examples: >>> GaussianSGD = DPOptimizerClassFactory(micro_batches=2) - >>> GaussianSGD.set_mechanisms('Gaussian', norm_bound=1.5, initial_noise_multiplier=5.0) - >>> net_opt = GaussianSGD.create('SGD')(params=network.trainable_params(), + >>> GaussianSGD.set_mechanisms('Gaussian', norm_bound=1.0, initial_noise_multiplier=1.5) + >>> net_opt = GaussianSGD.create('Momentum')(params=network.trainable_params(), >>> learning_rate=cfg.lr, >>> momentum=cfg.momentum) """ diff --git a/mindarmour/diff_privacy/train/model.py b/mindarmour/diff_privacy/train/model.py index 434bbe0152ccbb866e6c251c617cc42224b99e84..f84c29464a3c77a79422f39199c1dd1164ffd2a9 100644 --- a/mindarmour/diff_privacy/train/model.py +++ b/mindarmour/diff_privacy/train/model.py @@ -91,7 +91,7 @@ class DPModel(Model): >>> >>> net = Net() >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) - >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.01, momentum=0.9) >>> gaussian_mech = DPOptimizerClassFactory() >>> gaussian_mech.set_mechanisms('Gaussian', >>> norm_bound=args.l2_norm_bound,