first try of dcgan implementation

demo/gan/.gitignore

+output/
+*.png
+.pydevproject
+.project
+data/
+trainLog.txt
+

demo/gan/gan_conf_image.py

+# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from paddle.trainer_config_helpers import *
+from paddle.trainer_config_helpers.layers import img_convTrans_layer
+from paddle.trainer_config_helpers.activations import LinearActivation
+from numpy.distutils.system_info import tmp
+
+mode = get_config_arg("mode", str, "generator")
+assert mode in set(["generator",
+                    "discriminator",
+                    "generator_training",
+                    "discriminator_training"])
+
+is_generator_training = mode == "generator_training"
+is_discriminator_training = mode == "discriminator_training"
+is_generator = mode == "generator"
+is_discriminator = mode == "discriminator"
+
+print('mode=%s' % mode)
+noise_dim = 100
+gf_dim = 64
+df_dim = 64
+sample_dim = 28 # image dim
+c_dim = 1 # image color
+s2, s4 = int(sample_dim/2), int(sample_dim/4), 
+s8, s16 = int(sample_dim/8), int(sample_dim/16)
+
+settings(
+    batch_size=100,
+    learning_rate=1e-4,
+    learning_method=AdamOptimizer()
+)
+
+def convTrans_bn(input, channels, output_x, num_filters, imgSize, stride, name, 
+                 param_attr, bias_attr, param_attr_bn):
+    tmp =  imgSize - (output_x - 1) * stride
+    if tmp <= 1 or tmp > 5:
+        raise ValueError("convTrans input-output dimension does not fit")
+    elif tmp <= 3:
+        filter_size = tmp + 2
+        padding = 1
+    else:
+        filter_size = tmp
+        padding = 0
+        
+        
+    convTrans = img_convTrans_layer(input, filter_size=filter_size, 
+                   num_filters=num_filters,
+                   name=name + "_convt", num_channels=channels,
+                   act=LinearActivation(), groups=1, stride=stride, 
+                   padding=padding, bias_attr=bias_attr,
+                   param_attr=param_attr, shared_biases=True, layer_attr=None,
+                   filter_size_y=None, stride_y=None, padding_y=None)
+    
+    convTrans_bn = batch_norm_layer(convTrans, 
+                     act=ReluActivation(), 
+                     name=name + "_convt_bn", 
+                     bias_attr=bias_attr, 
+                     param_attr=param_attr_bn,
+                     use_global_stats=False)
+    
+    return convTrans_bn
+
+def conv_bn(input, channels, imgSize, num_filters, output_x, stride, name, 
+                 param_attr, bias_attr, param_attr_bn, bn):
+    tmp =  imgSize - (output_x - 1) * stride
+    if tmp <= 1 or tmp > 5:
+        raise ValueError("conv input-output dimension does not fit")
+    elif tmp <= 3:
+        filter_size = tmp + 2
+        padding = 1
+    else:
+        filter_size = tmp
+        padding = 0
+
+    print (imgSize, output_x, stride, filter_size, padding)
+        
+    if bn:
+        conv = img_conv_layer(input, filter_size=filter_size, 
+                   num_filters=num_filters,
+                   name=name + "_conv", num_channels=channels,
+                   act=LinearActivation(), groups=1, stride=stride, 
+                   padding=padding, bias_attr=bias_attr,
+                   param_attr=param_attr, shared_biases=True, layer_attr=None,
+                   filter_size_y=None, stride_y=None, padding_y=None)
+        
+        conv_bn = batch_norm_layer(conv, 
+                         act=ReluActivation(), 
+                         name=name + "_conv_bn", 
+                         bias_attr=bias_attr, 
+                         param_attr=param_attr_bn,
+                         use_global_stats=False)
+        
+        return conv_bn
+    else:
+        conv = img_conv_layer(input, filter_size=filter_size, 
+                   num_filters=num_filters,
+                   name=name + "_conv", num_channels=channels,
+                   act=ReluActivation(), groups=1, stride=stride, 
+                   padding=padding, bias_attr=bias_attr,
+                   param_attr=param_attr, shared_biases=True, layer_attr=None,
+                   filter_size_y=None, stride_y=None, padding_y=None)
+        return conv
+    
+def generator(noise):
+    """
+    generator generates a sample given noise
+    """
+    param_attr = ParamAttr(is_static=is_discriminator_training)
+    bias_attr = ParamAttr(is_static=is_discriminator_training,
+                           initial_mean=1.0,
+                           initial_std=0)
+    
+    param_attr_bn=ParamAttr(is_static=is_discriminator_training,
+                           initial_mean=1.0,
+                           initial_std=0.02)
+    
+    h1 = fc_layer(input=noise,
+                    name="gen_layer_h1",
+                    size=s8 * s8 * gf_dim * 4,
+                    bias_attr=bias_attr,
+                    param_attr=param_attr,
+                    #act=ReluActivation())
+                    act=LinearActivation())
+    
+    h1_bn = batch_norm_layer(h1, 
+                     act=ReluActivation(), 
+                     name="gen_layer_h1_bn", 
+                     bias_attr=bias_attr, 
+                     param_attr=param_attr_bn,
+                     use_global_stats=False)
+    
+    h2_bn = convTrans_bn(h1_bn, 
+                        channels=gf_dim*4, 
+                        output_x=s8,
+                        num_filters=gf_dim*2, 
+                        imgSize=s4,
+                        stride=2,
+                        name="gen_layer_h2", 
+                        param_attr=param_attr, 
+                        bias_attr=bias_attr, 
+                        param_attr_bn=param_attr_bn)
+    
+    h3_bn = convTrans_bn(h2_bn, 
+                        channels=gf_dim*2, 
+                        output_x=s4,
+                        num_filters=gf_dim, 
+                        imgSize=s2,
+                        stride=2,
+                        name="gen_layer_h3", 
+                        param_attr=param_attr, 
+                        bias_attr=bias_attr, 
+                        param_attr_bn=param_attr_bn)
+     
+    
+    return convTrans_bn(h3_bn,
+                        channels=gf_dim, 
+                        output_x=s2,
+                        num_filters=c_dim, 
+                        imgSize=sample_dim,
+                        stride=2,
+                        name="gen_layer_h4", 
+                        param_attr=param_attr, 
+                        bias_attr=bias_attr, 
+                        param_attr_bn=param_attr_bn)
+
+
+def discriminator(sample):
+    """
+    discriminator ouputs the probablity of a sample is from generator
+    or real data.
+    The output has two dimenstional: dimension 0 is the probablity
+    of the sample is from generator and dimension 1 is the probabblity
+    of the sample is from real data.
+    """
+    param_attr = ParamAttr(is_static=is_generator_training)
+    bias_attr = ParamAttr(is_static=is_generator_training,
+                          initial_mean=1.0,
+                          initial_std=0)
+    
+    param_attr_bn=ParamAttr(is_static=is_generator_training,
+                           initial_mean=1.0,
+                           initial_std=0.02)
+    
+    h0 = conv_bn(sample, 
+                 channels=c_dim, 
+                 imgSize=sample_dim,
+                 num_filters=df_dim, 
+                 output_x=s2, 
+                 stride=2, 
+                 name="dis_h0", 
+                 param_attr=param_attr, 
+                 bias_attr=bias_attr, 
+                 param_attr_bn=param_attr_bn, 
+                 bn=False)
+    
+    h1_bn = conv_bn(h0, 
+                 channels=df_dim,
+                 imgSize=s2,
+                 num_filters=df_dim*2, 
+                 output_x=s4, 
+                 stride=2, 
+                 name="dis_h1", 
+                 param_attr=param_attr, 
+                 bias_attr=bias_attr, 
+                 param_attr_bn=param_attr_bn, 
+                 bn=True)
+
+    h2_bn = conv_bn(h1_bn, 
+                 channels=df_dim*2,
+                 imgSize=s4,
+                 num_filters=df_dim*4, 
+                 output_x=s8, 
+                 stride=2, 
+                 name="dis_h2", 
+                 param_attr=param_attr, 
+                 bias_attr=bias_attr, 
+                 param_attr_bn=param_attr_bn, 
+                 bn=True)
+        
+    return fc_layer(input=h2_bn, name="dis_prob", size=2,
+                    bias_attr=bias_attr,
+                    param_attr=param_attr,
+                    act=SoftmaxActivation())
+
+
+
+if is_generator_training:
+    noise = data_layer(name="noise", size=noise_dim)
+    sample = generator(noise)
+
+if is_discriminator_training:
+    sample = data_layer(name="sample", size=sample_dim * sample_dim*c_dim)
+
+if is_generator_training or is_discriminator_training:
+    label = data_layer(name="label", size=1)
+    prob = discriminator(sample)
+    cost = cross_entropy(input=prob, label=label)
+    classification_error_evaluator(input=prob, label=label, name=mode+'_error')
+    outputs(cost)
+
+    
+if is_generator:
+    noise = data_layer(name="noise", size=noise_dim)
+    outputs(generator(noise))

demo/gan/gan_trainer_image.py

+# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import itertools
+import random
+import numpy
+import sys,os
+from PIL import Image
+
+from paddle.trainer.config_parser import parse_config
+from paddle.trainer.config_parser import logger
+import py_paddle.swig_paddle as api
+from py_paddle import DataProviderConverter
+
+import matplotlib.pyplot as plt
+
+
+def plot2DScatter(data, outputfile):
+    # Generate some test data
+    x = data[:, 0]
+    y = data[:, 1]
+    print "The mean vector is %s" % numpy.mean(data, 0)
+    print "The std vector is %s" % numpy.std(data, 0)
+
+    heatmap, xedges, yedges = numpy.histogram2d(x, y, bins=50)
+    extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]]
+
+    plt.clf()
+    plt.scatter(x, y)
+    # plt.show()
+    plt.savefig(outputfile, bbox_inches='tight')
+
+def CHECK_EQ(a, b):
+    assert a == b, "a=%s, b=%s" % (a, b)
+
+
+def copy_shared_parameters(src, dst):
+    src_params = [src.getParameter(i)
+               for i in xrange(src.getParameterSize())]
+    src_params = dict([(p.getName(), p) for p in src_params])
+
+
+    for i in xrange(dst.getParameterSize()):
+        dst_param = dst.getParameter(i)
+        src_param = src_params.get(dst_param.getName(), None)
+        if src_param is None:
+            continue
+        src_value = src_param.getBuf(api.PARAMETER_VALUE)
+        dst_value = dst_param.getBuf(api.PARAMETER_VALUE)
+        CHECK_EQ(len(src_value), len(dst_value))
+        dst_value.copyFrom(src_value)
+        dst_param.setValueUpdated()
+        
+def print_parameters(src):
+    src_params = [src.getParameter(i)
+               for i in xrange(src.getParameterSize())]
+
+    print "***************"
+    for p in src_params:
+        print "Name is %s" % p.getName()
+        print "value is %s \n" % p.getBuf(api.PARAMETER_VALUE).copyToNumpyArray()
+
+def load_mnist_data(imageFile):
+    f = open(imageFile, "rb")
+    f.read(16)
+
+    # Define number of samples for train/test
+    if "train" in imageFile:
+        #n = 60000
+        n = 60000
+    else:
+        n = 10000
+    
+    data = numpy.zeros((n, 28*28), dtype = "float32")
+    
+    for i in range(n):
+        pixels = []
+        for j in range(28 * 28):
+            pixels.append(float(ord(f.read(1))) / 255.0)
+        data[i, :] = pixels
+
+    f.close()
+    return data
+
+def saveImages(images, path):
+    for i in xrange(10):
+        im = Image.fromarray(images[i, :].reshape((28, 28)) * 255.0).convert('RGB')
+        im.save(path + "/image_" + str(i) + ".png")
+    
+def get_real_samples(batch_size, data_np):
+    return data_np[numpy.random.choice(data_np.shape[0], batch_size, 
+                                       replace=False),:]
+    
+def get_noise(batch_size, noise_dim):
+    return numpy.random.normal(size=(batch_size, noise_dim)).astype('float32')
+
+def get_fake_samples(generator_machine, batch_size, noise):
+    gen_inputs = prepare_generator_data_batch(batch_size, noise)
+    gen_inputs.resize(1)
+    gen_outputs = api.Arguments.createArguments(0)
+    generator_machine.forward(gen_inputs, gen_outputs, api.PASS_TEST)
+    fake_samples = gen_outputs.getSlotValue(0).copyToNumpyMat()
+    return fake_samples
+
+def get_training_loss(training_machine, inputs):
+    outputs = api.Arguments.createArguments(0)
+    training_machine.forward(inputs, outputs, api.PASS_TEST)
+    loss = outputs.getSlotValue(0).copyToNumpyMat()
+    return numpy.mean(loss)
+
+def prepare_discriminator_data_batch_pos(batch_size, data_np):
+    real_samples = get_real_samples(batch_size, data_np)
+    labels = numpy.ones(batch_size, dtype='int32')
+    inputs = api.Arguments.createArguments(2)
+    inputs.setSlotValue(0, api.Matrix.createCpuDenseFromNumpy(real_samples))
+    inputs.setSlotIds(1, api.IVector.createCpuVectorFromNumpy(labels))
+    return inputs
+
+def prepare_discriminator_data_batch_neg(generator_machine, batch_size, noise):
+    fake_samples = get_fake_samples(generator_machine, batch_size, noise)
+    #print fake_samples.shape
+    labels = numpy.zeros(batch_size, dtype='int32')
+    inputs = api.Arguments.createArguments(2)
+    inputs.setSlotValue(0, api.Matrix.createCpuDenseFromNumpy(fake_samples))
+    inputs.setSlotIds(1, api.IVector.createCpuVectorFromNumpy(labels))
+    return inputs
+
+def prepare_generator_data_batch(batch_size, noise):
+    label = numpy.ones(batch_size, dtype='int32')
+    #label = numpy.zeros(batch_size, dtype='int32')
+    inputs = api.Arguments.createArguments(2)
+    inputs.setSlotValue(0, api.Matrix.createCpuDenseFromNumpy(noise))
+    inputs.setSlotIds(1, api.IVector.createCpuVectorFromNumpy(label))
+    return inputs
+
+
+def find(iterable, cond):
+    for item in iterable:
+        if cond(item):
+            return item
+    return None
+
+
+def get_layer_size(model_conf, layer_name):
+    layer_conf = find(model_conf.layers, lambda x: x.name == layer_name)
+    assert layer_conf is not None, "Cannot find '%s' layer" % layer_name
+    return layer_conf.size
+
+
+def main():
+    api.initPaddle('--use_gpu=0', '--dot_period=10', '--log_period=100')
+    gen_conf = parse_config("gan_conf_image.py", "mode=generator_training")
+    dis_conf = parse_config("gan_conf_image.py", "mode=discriminator_training")
+    generator_conf = parse_config("gan_conf_image.py", "mode=generator")
+    batch_size = dis_conf.opt_config.batch_size
+    noise_dim = get_layer_size(gen_conf.model_config, "noise")
+    sample_dim = get_layer_size(dis_conf.model_config, "sample")
+    
+    data_np = load_mnist_data("./data/raw_data/train-images-idx3-ubyte")
+
+    # this create a gradient machine for discriminator
+    dis_training_machine = api.GradientMachine.createFromConfigProto(
+        dis_conf.model_config)
+
+    gen_training_machine = api.GradientMachine.createFromConfigProto(
+        gen_conf.model_config)
+
+    # generator_machine is used to generate data only, which is used for
+    # training discrinator
+    logger.info(str(generator_conf.model_config))
+    generator_machine = api.GradientMachine.createFromConfigProto(
+        generator_conf.model_config)
+    
+    dis_trainer = api.Trainer.create(
+        dis_conf, dis_training_machine)
+
+    gen_trainer = api.Trainer.create(
+        gen_conf, gen_training_machine)
+    
+    dis_trainer.startTrain()
+    gen_trainer.startTrain()
+    
+    copy_shared_parameters(gen_training_machine, dis_training_machine)
+    copy_shared_parameters(gen_training_machine, generator_machine)
+    
+    curr_train = "dis"
+    curr_strike = 0
+    MAX_strike = 100
+     
+    for train_pass in xrange(100):
+        dis_trainer.startTrainPass()
+        gen_trainer.startTrainPass()
+        for i in xrange(1000):
+#             data_batch_dis = prepare_discriminator_data_batch(
+#                     generator_machine, batch_size, noise_dim, sample_dim)
+#             dis_loss = get_training_loss(dis_training_machine, data_batch_dis)
+            noise = get_noise(batch_size, noise_dim)
+            data_batch_dis_pos = prepare_discriminator_data_batch_pos(
+                batch_size, data_np)
+            dis_loss_pos = get_training_loss(dis_training_machine, data_batch_dis_pos)
+                        
+            data_batch_dis_neg = prepare_discriminator_data_batch_neg(
+                generator_machine, batch_size, noise)
+            dis_loss_neg = get_training_loss(dis_training_machine, data_batch_dis_neg)            
+                         
+            dis_loss = (dis_loss_pos + dis_loss_neg) / 2.0
+             
+            data_batch_gen = prepare_generator_data_batch(
+                    batch_size, noise)
+            gen_loss = get_training_loss(gen_training_machine, data_batch_gen)
+             
+            if i % 100 == 0:
+                print "d_pos_loss is %s     d_neg_loss is %s" % (dis_loss_pos, dis_loss_neg) 
+                print "d_loss is %s    g_loss is %s" % (dis_loss, gen_loss)
+                             
+            if (not (curr_train == "dis" and curr_strike == MAX_strike)) and ((curr_train == "gen" and curr_strike == MAX_strike) or dis_loss > gen_loss):
+                if curr_train == "dis":
+                    curr_strike += 1
+                else:
+                    curr_train = "dis"
+                    curr_strike = 1                
+                dis_trainer.trainOneDataBatch(batch_size, data_batch_dis_neg)
+                dis_trainer.trainOneDataBatch(batch_size, data_batch_dis_pos)
+#                 dis_loss = numpy.mean(dis_trainer.getForwardOutput()[0]["value"])
+#                 print "getForwardOutput loss is %s" % dis_loss                
+                copy_shared_parameters(dis_training_machine, gen_training_machine)
+ 
+            else:
+                if curr_train == "gen":
+                    curr_strike += 1
+                else:
+                    curr_train = "gen"
+                    curr_strike = 1
+                gen_trainer.trainOneDataBatch(batch_size, data_batch_gen)    
+                copy_shared_parameters(gen_training_machine, dis_training_machine)
+                copy_shared_parameters(gen_training_machine, generator_machine)
+ 
+        dis_trainer.finishTrainPass()
+        gen_trainer.finishTrainPass()
+        
+        
+        fake_samples = get_fake_samples(generator_machine, batch_size, noise)
+        save_dir = "./pass_" + str(train_pass)
+        if not os.path.exists(save_dir):
+            os.makedirs(save_dir)
+        saveImages(fake_samples, save_dir)
+    dis_trainer.finishTrain()
+    gen_trainer.finishTrain()
+
+if __name__ == '__main__':
+    main()