From de27569ecaf1c39e4dd11afa061c51888911b9d9 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Tue, 30 Jun 2020 17:22:58 +0800 Subject: [PATCH] [Dy2Stat] fix diff of cycle GAN model on GPU (#25233) * fix GPU diff test=develop * refine code test=develop --- .../dygraph_to_static/test_cycle_gan.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cycle_gan.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cycle_gan.py index 844438eaf6..3578dc58f1 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cycle_gan.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cycle_gan.py @@ -40,10 +40,13 @@ from paddle.fluid.dygraph.nn import Conv2D, Conv2DTranspose, BatchNorm # Note: Set True to eliminate randomness. # 1. For one operation, cuDNN has several algorithms, # some algorithm results are non-deterministic, like convolution algorithms. +# 2. If include BatchNorm, please set `use_global_stats=True` to avoid using +# cudnnBatchNormalizationBackward which is non-deterministic. if fluid.is_compiled_with_cuda(): fluid.set_flags({'FLAGS_cudnn_deterministic': True}) -use_cudnn = True +# set False to speed up training. +use_cudnn = False step_per_epoch = 10 lambda_A = 10.0 lambda_B = 10.0 @@ -110,7 +113,7 @@ class Cycle_Gan(fluid.dygraph.Layer): return fake_A, fake_B, cyc_A, cyc_B, g_A_loss, g_B_loss, idt_loss_A, idt_loss_B, cyc_A_loss, cyc_B_loss, g_loss @declarative - def disriminatorA(self, input_A, input_B): + def discriminatorA(self, input_A, input_B): """ Discriminator A of GAN model. """ @@ -326,6 +329,7 @@ class conv2d(fluid.dygraph.Layer): bias_attr=con_bias_attr) if norm: self.bn = BatchNorm( + use_global_stats=True, # set True to use deterministic algorithm num_channels=num_filters, param_attr=fluid.ParamAttr( initializer=fluid.initializer.NormalInitializer(1.0, 0.02)), @@ -381,6 +385,7 @@ class DeConv2D(fluid.dygraph.Layer): bias_attr=de_bias_attr) if norm: self.bn = BatchNorm( + use_global_stats=True, # set True to use deterministic algorithm num_channels=num_filters, param_attr=fluid.ParamAttr( initializer=fluid.initializer.NormalInitializer(1.0, 0.02)), @@ -429,7 +434,6 @@ class ImagePool(object): def reader_creater(): - # local_random = np.random.RandomState(SEED) def reader(): while True: fake_image = np.uint8( @@ -480,13 +484,8 @@ def optimizer_setting(parameters): def train(args, to_static): - # FIXME(Aurelius84): Found diff just on GPU and it disappears when we remove the BatchNorm layers. - # In dygraph mode, it still exists with different output while executing the every time. - - # place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ - # else fluid.CPUPlace() - - place = fluid.CPUPlace() + place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ + else fluid.CPUPlace() program_translator.enable(to_static) @@ -553,8 +552,8 @@ def train(args, to_static): fake_pool_A = to_variable(fake_pool_A) # optimize the d_A network - rec_B, fake_pool_rec_B = cycle_gan.disriminatorA(data_B, - fake_pool_B) + rec_B, fake_pool_rec_B = cycle_gan.discriminatorA(data_B, + fake_pool_B) d_loss_A = (fluid.layers.square(fake_pool_rec_B) + fluid.layers.square(rec_B - 1)) / 2.0 d_loss_A = fluid.layers.reduce_mean(d_loss_A) @@ -581,7 +580,6 @@ def train(args, to_static): idt_loss_A, g_B_loss, cyc_B_loss, idt_loss_B ] cur_batch_loss = [x.numpy()[0] for x in cur_batch_loss] - loss_data.append(cur_batch_loss) batch_time = time.time() - s_time t_time += batch_time @@ -593,6 +591,7 @@ def train(args, to_static): if batch_id > args.train_step: break + loss_data.append(cur_batch_loss) return np.array(loss_data) -- GitLab