change PyReader to from_generator (#3490)

* change PyReader to from_generator

* fluid.layers.data->fluid.data

* fix_bug

PaddleCV/PaddleGAN/data_reader.py

@@ -47,7 +47,7 @@ def RandomHorizonFlip(img):
     return img
 
 
-def get_preprocess_param(load_size, crop_size):
+def get_preprocess_param2(load_size, crop_size):
     x = np.random.randint(0, np.maximum(0, load_size - crop_size))
     y = np.random.randint(0, np.maximum(0, load_size - crop_size))
     flip = np.random.rand() > 0.5
@@ -59,7 +59,7 @@ def get_preprocess_param(load_size, crop_size):
     }
 
 
-def get_preprocess_param(load_width, load_height, crop_width, crop_height):
+def get_preprocess_param4(load_width, load_height, crop_width, crop_height):
     if crop_width == load_width:
         x = 0
         y = 0
@@ -178,7 +178,7 @@ class pair_reader_creator(reader_creator):
                 self.id2name[i] = os.path.basename(files[0])
 
                 if self.mode == "TRAIN":
-                    param = get_preprocess_param(args.image_size,
+                    param = get_preprocess_param2(args.image_size,
                                                   args.crop_size)
                     img1 = img1.resize((args.image_size, args.image_size),
                                        Image.BICUBIC)
@@ -266,7 +266,7 @@ class triplex_reader_creator(reader_creator):
                     img3 = Image.open(os.path.join(self.image_dir, files[2]))
 
                 if self.mode == "TRAIN":
-                    param = get_preprocess_param(
+                    param = get_preprocess_param4(
                         args.load_width, args.load_height, args.crop_width,
                         args.crop_height)
                     img1 = img1.resize((args.load_width, args.load_height),

PaddleCV/PaddleGAN/infer.py

@@ -83,11 +83,12 @@ def infer(args):
 
     model_name = 'net_G'
     if args.model_net == 'CycleGAN':
-        py_reader = fluid.io.PyReader(
+        loader = fluid.io.DataLoader.from_generator(
             feed_list=[input, image_name],
             capacity=4,  ## batch_size * 4
             iterable=True,
             use_double_buffer=True)
+
         from network.CycleGAN_network import CycleGAN_model
         model = CycleGAN_model()
         if args.input_style == "A":
@@ -97,7 +98,7 @@ def infer(args):
         else:
             raise "Input with style [%s] is not supported." % args.input_style
     elif args.model_net == 'Pix2pix':
-        py_reader = fluid.io.PyReader(
+        loader = fluid.io.DataLoader.from_generator(
             feed_list=[input, image_name],
             capacity=4,  ## batch_size * 4
             iterable=True,
@@ -295,11 +296,11 @@ def infer(args):
             batch_size=args.n_samples,
             mode="VAL")
         reader_test = test_reader.make_reader(args, return_name=True)
-        py_reader.decorate_batch_generator(
+        loader.set_batch_generator(
             reader_test,
             places=fluid.cuda_places() if args.use_gpu else fluid.cpu_places())
         id2name = test_reader.id2name
-        for data in py_reader():
+        for data in loader():
             real_img, image_name = data[0]['input'], data[0]['image_name']
             image_name = id2name[np.array(image_name).astype('int32')[0]]
             print("read: ", image_name)

PaddleCV/PaddleGAN/trainer/AttGAN.py

@@ -272,16 +272,16 @@ class AttGAN(object):
     def build_model(self):
         data_shape = [-1, 3, self.cfg.image_size, self.cfg.image_size]
 
-        image_real = fluid.layers.data(
+        image_real = fluid.data(
             name='image_real', shape=data_shape, dtype='float32')
-        label_org = fluid.layers.data(
-            name='label_org', shape=[self.cfg.c_dim], dtype='float32')
-        label_trg = fluid.layers.data(
-            name='label_trg', shape=[self.cfg.c_dim], dtype='float32')
-        label_org_ = fluid.layers.data(
-            name='label_org_', shape=[self.cfg.c_dim], dtype='float32')
-        label_trg_ = fluid.layers.data(
-            name='label_trg_', shape=[self.cfg.c_dim], dtype='float32')
+        label_org = fluid.data(
+            name='label_org', shape=[-1, self.cfg.c_dim], dtype='float32')
+        label_trg = fluid.data(
+            name='label_trg', shape=[-1, self.cfg.c_dim], dtype='float32')
+        label_org_ = fluid.data(
+            name='label_org_', shape=[-1, self.cfg.c_dim], dtype='float32')
+        label_trg_ = fluid.data(
+            name='label_trg_', shape=[-1, self.cfg.c_dim], dtype='float32')
 
         py_reader = fluid.io.PyReader(
             feed_list=[image_real, label_org, label_trg],

PaddleCV/PaddleGAN/trainer/CGAN.py

@@ -88,12 +88,11 @@ class CGAN(object):
 
     def build_model(self):
 
-        img = fluid.layers.data(name='img', shape=[784], dtype='float32')
-        condition = fluid.layers.data(
-            name='condition', shape=[1], dtype='float32')
-        noise = fluid.layers.data(
-            name='noise', shape=[self.cfg.noise_size], dtype='float32')
-        label = fluid.layers.data(name='label', shape=[1], dtype='float32')
+        img = fluid.data(name='img', shape=[-1, 784], dtype='float32')
+        condition = fluid.data(name='condition', shape=[-1, 1], dtype='float32')
+        noise = fluid.data(
+            name='noise', shape=[-1, self.cfg.noise_size], dtype='float32')
+        label = fluid.data(name='label', shape=[-1, 1], dtype='float32')
 
         g_trainer = GTrainer(noise, condition, self.cfg)
         d_trainer = DTrainer(img, condition, label, self.cfg)

PaddleCV/PaddleGAN/trainer/CycleGAN.py

@@ -231,13 +231,11 @@ class CycleGAN(object):
     def build_model(self):
         data_shape = [-1, 3, self.cfg.crop_size, self.cfg.crop_size]
 
-        input_A = fluid.layers.data(
-            name='input_A', shape=data_shape, dtype='float32')
-        input_B = fluid.layers.data(
-            name='input_B', shape=data_shape, dtype='float32')
-        fake_pool_A = fluid.layers.data(
+        input_A = fluid.data(name='input_A', shape=data_shape, dtype='float32')
+        input_B = fluid.data(name='input_B', shape=data_shape, dtype='float32')
+        fake_pool_A = fluid.data(
             name='fake_pool_A', shape=data_shape, dtype='float32')
-        fake_pool_B = fluid.layers.data(
+        fake_pool_B = fluid.data(
             name='fake_pool_B', shape=data_shape, dtype='float32')
 
         A_py_reader = fluid.io.PyReader(
@@ -348,10 +346,10 @@ class CycleGAN(object):
                 batch_id += 1
 
             if self.cfg.run_test:
-                A_image_name = fluid.layers.data(
-                    name='A_image_name', shape=[1], dtype='int32')
-                B_image_name = fluid.layers.data(
-                    name='B_image_name', shape=[1], dtype='int32')
+                A_image_name = fluid.data(
+                    name='A_image_name', shape=[-1, 1], dtype='int32')
+                B_image_name = fluid.data(
+                    name='B_image_name', shape=[-1, 1], dtype='int32')
                 A_test_py_reader = fluid.io.PyReader(
                     feed_list=[input_A, A_image_name],
                     capacity=4,

PaddleCV/PaddleGAN/trainer/DCGAN.py

@@ -86,10 +86,10 @@ class DCGAN(object):
         self.train_reader = train_reader
 
     def build_model(self):
-        img = fluid.layers.data(name='img', shape=[784], dtype='float32')
-        noise = fluid.layers.data(
-            name='noise', shape=[self.cfg.noise_size], dtype='float32')
-        label = fluid.layers.data(name='label', shape=[1], dtype='float32')
+        img = fluid.data(name='img', shape=[-1, 784], dtype='float32')
+        noise = fluid.data(
+            name='noise', shape=[-1, self.cfg.noise_size], dtype='float32')
+        label = fluid.data(name='label', shape=[-1, 1], dtype='float32')
 
         g_trainer = GTrainer(noise, label, self.cfg)
         d_trainer = DTrainer(img, label, self.cfg)

PaddleCV/PaddleGAN/trainer/Pix2pix.py

@@ -213,16 +213,14 @@ class Pix2pix(object):
     def build_model(self):
         data_shape = [-1, 3, self.cfg.crop_size, self.cfg.crop_size]
 
-        input_A = fluid.layers.data(
-            name='input_A', shape=data_shape, dtype='float32')
-        input_B = fluid.layers.data(
-            name='input_B', shape=data_shape, dtype='float32')
-        input_fake = fluid.layers.data(
+        input_A = fluid.data(name='input_A', shape=data_shape, dtype='float32')
+        input_B = fluid.data(name='input_B', shape=data_shape, dtype='float32')
+        input_fake = fluid.data(
             name='input_fake', shape=data_shape, dtype='float32')
 
-        py_reader = fluid.io.PyReader(
+        loader = fluid.io.DataLoader.from_generator(
             feed_list=[input_A, input_B],
-            capacity=4,  ## batch_size * 4
+            capacity=4,
             iterable=True,
             use_double_buffer=True)
 
@@ -232,7 +230,7 @@ class Pix2pix(object):
 
         # prepare environment
         place = fluid.CUDAPlace(0) if self.cfg.use_gpu else fluid.CPUPlace()
-        py_reader.decorate_batch_generator(
+        loader.set_batch_generator(
             self.train_reader,
             places=fluid.cuda_places()
             if self.cfg.use_gpu else fluid.cpu_places())
@@ -259,7 +257,7 @@ class Pix2pix(object):
 
         for epoch_id in range(self.cfg.epoch):
             batch_id = 0
-            for tensor in py_reader():
+            for tensor in loader():
                 s_time = time.time()
 
                 tensor_A, tensor_B = tensor[0]['input_A'], tensor[0]['input_B']
@@ -298,16 +296,16 @@ class Pix2pix(object):
                 batch_id += 1
 
             if self.cfg.run_test:
-                image_name = fluid.layers.data(
+                image_name = fluid.data(
                     name='image_name',
-                    shape=[self.cfg.batch_size],
+                    shape=[-1, self.cfg.batch_size],
                     dtype="int32")
-                test_py_reader = fluid.io.PyReader(
+                test_loader = fluid.io.DataLoader.from_generator(
                     feed_list=[input_A, input_B, image_name],
-                    capacity=4,  ## batch_size * 4
+                    capacity=4,
                     iterable=True,
                     use_double_buffer=True)
-                test_py_reader.decorate_batch_generator(
+                test_loader.set_batch_generator(
                     self.test_reader,
                     places=fluid.cuda_places()
                     if self.cfg.use_gpu else fluid.cpu_places())
@@ -319,7 +317,7 @@ class Pix2pix(object):
                     place,
                     test_program,
                     gen_trainer,
-                    test_py_reader,
+                    test_loader,
                     A_id2name=self.id2name)
 
             if self.cfg.save_checkpoints:

PaddleCV/PaddleGAN/trainer/SPADE.py

@@ -289,13 +289,13 @@ class SPADE(object):
         ]
         edge_shape = [-1, 1, self.cfg.crop_height, self.cfg.crop_width]
 
-        input_A = fluid.layers.data(
+        input_A = fluid.data(
             name='input_label', shape=label_shape, dtype='float32')
-        input_B = fluid.layers.data(
+        input_B = fluid.data(
             name='input_img', shape=data_shape, dtype='float32')
-        input_C = fluid.layers.data(
+        input_C = fluid.data(
             name='input_ins', shape=edge_shape, dtype='float32')
-        input_fake = fluid.layers.data(
+        input_fake = fluid.data(
             name='input_fake', shape=data_shape, dtype='float32')
 
         gen_trainer = GTrainer(input_A, input_B, input_C, self.cfg,
@@ -325,7 +325,7 @@ class SPADE(object):
 
         ### memory optim
         build_strategy = fluid.BuildStrategy()
-        build_strategy.enable_inplace = False
+        build_strategy.enable_inplace = True
         build_strategy.sync_batch_norm = False
 
         gen_trainer_program = fluid.CompiledProgram(
@@ -387,9 +387,9 @@ class SPADE(object):
 
             if self.cfg.run_test:
                 test_program = gen_trainer.infer_program
-                image_name = fluid.layers.data(
+                image_name = fluid.data(
                     name='image_name',
-                    shape=[self.cfg.batch_size],
+                    shape=[-1, self.cfg.batch_size],
                     dtype="int32")
                 test_py_reader = fluid.io.PyReader(
                     feed_list=[input_A, input_B, input_C, image_name],

PaddleCV/PaddleGAN/trainer/STGAN.py

@@ -284,16 +284,16 @@ class STGAN(object):
     def build_model(self):
         data_shape = [-1, 3, self.cfg.image_size, self.cfg.image_size]
 
-        image_real = fluid.layers.data(
+        image_real = fluid.data(
             name='image_real', shape=data_shape, dtype='float32')
-        label_org = fluid.layers.data(
-            name='label_org', shape=[self.cfg.c_dim], dtype='float32')
-        label_trg = fluid.layers.data(
-            name='label_trg', shape=[self.cfg.c_dim], dtype='float32')
-        label_org_ = fluid.layers.data(
-            name='label_org_', shape=[self.cfg.c_dim], dtype='float32')
-        label_trg_ = fluid.layers.data(
-            name='label_trg_', shape=[self.cfg.c_dim], dtype='float32')
+        label_org = fluid.data(
+            name='label_org', shape=[-1, self.cfg.c_dim], dtype='float32')
+        label_trg = fluid.data(
+            name='label_trg', shape=[-1, self.cfg.c_dim], dtype='float32')
+        label_org_ = fluid.data(
+            name='label_org_', shape=[-1, self.cfg.c_dim], dtype='float32')
+        label_trg_ = fluid.data(
+            name='label_trg_', shape=[-1, self.cfg.c_dim], dtype='float32')
 
         test_gen_trainer = GTrainer(image_real, label_org, label_org_,
                                     label_trg, label_trg_, self.cfg,
@@ -378,9 +378,9 @@ class STGAN(object):
                 batch_id += 1
 
             if self.cfg.run_test:
-                image_name = fluid.layers.data(
+                image_name = fluid.data(
                     name='image_name',
-                    shape=[self.cfg.n_samples],
+                    shape=[-1, self.cfg.n_samples],
                     dtype='int32')
                 test_py_reader = fluid.io.PyReader(
                     feed_list=[image_real, label_org, label_trg, image_name],

PaddleCV/PaddleGAN/trainer/StarGAN.py

@@ -261,12 +261,12 @@ class StarGAN(object):
     def build_model(self):
         data_shape = [-1, 3, self.cfg.image_size, self.cfg.image_size]
 
-        image_real = fluid.layers.data(
+        image_real = fluid.data(
             name='image_real', shape=data_shape, dtype='float32')
-        label_org = fluid.layers.data(
-            name='label_org', shape=[self.cfg.c_dim], dtype='float32')
-        label_trg = fluid.layers.data(
-            name='label_trg', shape=[self.cfg.c_dim], dtype='float32')
+        label_org = fluid.data(
+            name='label_org', shape=[-1, self.cfg.c_dim], dtype='float32')
+        label_trg = fluid.data(
+            name='label_trg', shape=[-1, self.cfg.c_dim], dtype='float32')
 
         py_reader = fluid.io.PyReader(
             feed_list=[image_real, label_org, label_trg],
@@ -346,9 +346,9 @@ class StarGAN(object):
                 batch_id += 1
 
             if self.cfg.run_test:
-                image_name = fluid.layers.data(
+                image_name = fluid.data(
                     name='image_name',
-                    shape=[self.cfg.n_samples],
+                    shape=[-1, self.cfg.n_samples],
                     dtype='int32')
                 test_py_reader = fluid.io.PyReader(
                     feed_list=[image_real, label_org, label_trg, image_name],

PaddleCV/PaddleGAN/util/utility.py

@@ -150,7 +150,7 @@ def save_test_image(epoch,
                                         "input_B": B_data})
             fake_B_temp = np.squeeze(fake_B_temp[0]).transpose([1, 2, 0])
             input_A_temp = np.squeeze(np.array(A_data)[0]).transpose([1, 2, 0])
-            input_B_temp = np.squeeze(np.array(A_data)[0]).transpose([1, 2, 0])
+            input_B_temp = np.squeeze(np.array(B_data)[0]).transpose([1, 2, 0])
 
             fakeB_name = "fakeB_" + str(epoch) + "_" + A_id2name[np.array(
                 image_name).astype('int32')[0]]