Revert "change PyReader to from_generator" (#3548)

* Revert "change PyReader to from_generator (#3490)"

This reverts commit d331b799.

* Update data_reader.py

* Update data_reader.py

* Update utility.py

PaddleCV/PaddleGAN/infer.py

@@ -83,12 +83,11 @@ def infer(args):
 
     model_name = 'net_G'
     if args.model_net == 'CycleGAN':
-        loader = fluid.io.DataLoader.from_generator(
+        py_reader = fluid.io.PyReader(
             feed_list=[input, image_name],
             capacity=4,  ## batch_size * 4
             iterable=True,
             use_double_buffer=True)
-
         from network.CycleGAN_network import CycleGAN_model
         model = CycleGAN_model()
         if args.input_style == "A":
@@ -98,7 +97,7 @@ def infer(args):
         else:
             raise "Input with style [%s] is not supported." % args.input_style
     elif args.model_net == 'Pix2pix':
-        loader = fluid.io.DataLoader.from_generator(
+        py_reader = fluid.io.PyReader(
             feed_list=[input, image_name],
             capacity=4,  ## batch_size * 4
             iterable=True,
@@ -296,11 +295,11 @@ def infer(args):
             batch_size=args.n_samples,
             mode="VAL")
         reader_test = test_reader.make_reader(args, return_name=True)
-        loader.set_batch_generator(
+        py_reader.decorate_batch_generator(
             reader_test,
             places=fluid.cuda_places() if args.use_gpu else fluid.cpu_places())
         id2name = test_reader.id2name
-        for data in loader():
+        for data in py_reader():
             real_img, image_name = data[0]['input'], data[0]['image_name']
             image_name = id2name[np.array(image_name).astype('int32')[0]]
             print("read: ", image_name)

PaddleCV/PaddleGAN/trainer/AttGAN.py

@@ -272,16 +272,16 @@ class AttGAN(object):
     def build_model(self):
         data_shape = [-1, 3, self.cfg.image_size, self.cfg.image_size]
 
-        image_real = fluid.data(
+        image_real = fluid.layers.data(
             name='image_real', shape=data_shape, dtype='float32')
-        label_org = fluid.data(
-            name='label_org', shape=[-1, self.cfg.c_dim], dtype='float32')
-        label_trg = fluid.data(
-            name='label_trg', shape=[-1, self.cfg.c_dim], dtype='float32')
-        label_org_ = fluid.data(
-            name='label_org_', shape=[-1, self.cfg.c_dim], dtype='float32')
-        label_trg_ = fluid.data(
-            name='label_trg_', shape=[-1, self.cfg.c_dim], dtype='float32')
+        label_org = fluid.layers.data(
+            name='label_org', shape=[self.cfg.c_dim], dtype='float32')
+        label_trg = fluid.layers.data(
+            name='label_trg', shape=[self.cfg.c_dim], dtype='float32')
+        label_org_ = fluid.layers.data(
+            name='label_org_', shape=[self.cfg.c_dim], dtype='float32')
+        label_trg_ = fluid.layers.data(
+            name='label_trg_', shape=[self.cfg.c_dim], dtype='float32')
 
         py_reader = fluid.io.PyReader(
             feed_list=[image_real, label_org, label_trg],

PaddleCV/PaddleGAN/trainer/CGAN.py

@@ -88,11 +88,12 @@ class CGAN(object):
 
     def build_model(self):
 
-        img = fluid.data(name='img', shape=[-1, 784], dtype='float32')
-        condition = fluid.data(name='condition', shape=[-1, 1], dtype='float32')
-        noise = fluid.data(
-            name='noise', shape=[-1, self.cfg.noise_size], dtype='float32')
-        label = fluid.data(name='label', shape=[-1, 1], dtype='float32')
+        img = fluid.layers.data(name='img', shape=[784], dtype='float32')
+        condition = fluid.layers.data(
+            name='condition', shape=[1], dtype='float32')
+        noise = fluid.layers.data(
+            name='noise', shape=[self.cfg.noise_size], dtype='float32')
+        label = fluid.layers.data(name='label', shape=[1], dtype='float32')
 
         g_trainer = GTrainer(noise, condition, self.cfg)
         d_trainer = DTrainer(img, condition, label, self.cfg)

PaddleCV/PaddleGAN/trainer/CycleGAN.py

@@ -231,11 +231,13 @@ class CycleGAN(object):
     def build_model(self):
         data_shape = [-1, 3, self.cfg.crop_size, self.cfg.crop_size]
 
-        input_A = fluid.data(name='input_A', shape=data_shape, dtype='float32')
-        input_B = fluid.data(name='input_B', shape=data_shape, dtype='float32')
-        fake_pool_A = fluid.data(
+        input_A = fluid.layers.data(
+            name='input_A', shape=data_shape, dtype='float32')
+        input_B = fluid.layers.data(
+            name='input_B', shape=data_shape, dtype='float32')
+        fake_pool_A = fluid.layers.data(
             name='fake_pool_A', shape=data_shape, dtype='float32')
-        fake_pool_B = fluid.data(
+        fake_pool_B = fluid.layers.data(
             name='fake_pool_B', shape=data_shape, dtype='float32')
 
         A_py_reader = fluid.io.PyReader(
@@ -346,10 +348,10 @@ class CycleGAN(object):
                 batch_id += 1
 
             if self.cfg.run_test:
-                A_image_name = fluid.data(
-                    name='A_image_name', shape=[-1, 1], dtype='int32')
-                B_image_name = fluid.data(
-                    name='B_image_name', shape=[-1, 1], dtype='int32')
+                A_image_name = fluid.layers.data(
+                    name='A_image_name', shape=[1], dtype='int32')
+                B_image_name = fluid.layers.data(
+                    name='B_image_name', shape=[1], dtype='int32')
                 A_test_py_reader = fluid.io.PyReader(
                     feed_list=[input_A, A_image_name],
                     capacity=4,

PaddleCV/PaddleGAN/trainer/DCGAN.py

@@ -86,10 +86,10 @@ class DCGAN(object):
         self.train_reader = train_reader
 
     def build_model(self):
-        img = fluid.data(name='img', shape=[-1, 784], dtype='float32')
-        noise = fluid.data(
-            name='noise', shape=[-1, self.cfg.noise_size], dtype='float32')
-        label = fluid.data(name='label', shape=[-1, 1], dtype='float32')
+        img = fluid.layers.data(name='img', shape=[784], dtype='float32')
+        noise = fluid.layers.data(
+            name='noise', shape=[self.cfg.noise_size], dtype='float32')
+        label = fluid.layers.data(name='label', shape=[1], dtype='float32')
 
         g_trainer = GTrainer(noise, label, self.cfg)
         d_trainer = DTrainer(img, label, self.cfg)

PaddleCV/PaddleGAN/trainer/Pix2pix.py

@@ -213,14 +213,16 @@ class Pix2pix(object):
     def build_model(self):
         data_shape = [-1, 3, self.cfg.crop_size, self.cfg.crop_size]
 
-        input_A = fluid.data(name='input_A', shape=data_shape, dtype='float32')
-        input_B = fluid.data(name='input_B', shape=data_shape, dtype='float32')
-        input_fake = fluid.data(
+        input_A = fluid.layers.data(
+            name='input_A', shape=data_shape, dtype='float32')
+        input_B = fluid.layers.data(
+            name='input_B', shape=data_shape, dtype='float32')
+        input_fake = fluid.layers.data(
             name='input_fake', shape=data_shape, dtype='float32')
 
-        loader = fluid.io.DataLoader.from_generator(
+        py_reader = fluid.io.PyReader(
             feed_list=[input_A, input_B],
-            capacity=4,
+            capacity=4,  ## batch_size * 4
             iterable=True,
             use_double_buffer=True)
 
@@ -230,7 +232,7 @@ class Pix2pix(object):
 
         # prepare environment
         place = fluid.CUDAPlace(0) if self.cfg.use_gpu else fluid.CPUPlace()
-        loader.set_batch_generator(
+        py_reader.decorate_batch_generator(
             self.train_reader,
             places=fluid.cuda_places()
             if self.cfg.use_gpu else fluid.cpu_places())
@@ -257,7 +259,7 @@ class Pix2pix(object):
 
         for epoch_id in range(self.cfg.epoch):
             batch_id = 0
-            for tensor in loader():
+            for tensor in py_reader():
                 s_time = time.time()
 
                 tensor_A, tensor_B = tensor[0]['input_A'], tensor[0]['input_B']
@@ -296,16 +298,16 @@ class Pix2pix(object):
                 batch_id += 1
 
             if self.cfg.run_test:
-                image_name = fluid.data(
+                image_name = fluid.layers.data(
                     name='image_name',
-                    shape=[-1, self.cfg.batch_size],
+                    shape=[self.cfg.batch_size],
                     dtype="int32")
-                test_loader = fluid.io.DataLoader.from_generator(
+                test_py_reader = fluid.io.PyReader(
                     feed_list=[input_A, input_B, image_name],
-                    capacity=4,
+                    capacity=4,  ## batch_size * 4
                     iterable=True,
                     use_double_buffer=True)
-                test_loader.set_batch_generator(
+                test_py_reader.decorate_batch_generator(
                     self.test_reader,
                     places=fluid.cuda_places()
                     if self.cfg.use_gpu else fluid.cpu_places())
@@ -317,7 +319,7 @@ class Pix2pix(object):
                     place,
                     test_program,
                     gen_trainer,
-                    test_loader,
+                    test_py_reader,
                     A_id2name=self.id2name)
 
             if self.cfg.save_checkpoints:

PaddleCV/PaddleGAN/trainer/SPADE.py

@@ -289,13 +289,13 @@ class SPADE(object):
         ]
         edge_shape = [-1, 1, self.cfg.crop_height, self.cfg.crop_width]
 
-        input_A = fluid.data(
+        input_A = fluid.layers.data(
             name='input_label', shape=label_shape, dtype='float32')
-        input_B = fluid.data(
+        input_B = fluid.layers.data(
             name='input_img', shape=data_shape, dtype='float32')
-        input_C = fluid.data(
+        input_C = fluid.layers.data(
             name='input_ins', shape=edge_shape, dtype='float32')
-        input_fake = fluid.data(
+        input_fake = fluid.layers.data(
             name='input_fake', shape=data_shape, dtype='float32')
 
         gen_trainer = GTrainer(input_A, input_B, input_C, self.cfg,
@@ -325,7 +325,7 @@ class SPADE(object):
 
         ### memory optim
         build_strategy = fluid.BuildStrategy()
-        build_strategy.enable_inplace = True
+        build_strategy.enable_inplace = False
         build_strategy.sync_batch_norm = False
 
         gen_trainer_program = fluid.CompiledProgram(
@@ -387,9 +387,9 @@ class SPADE(object):
 
             if self.cfg.run_test:
                 test_program = gen_trainer.infer_program
-                image_name = fluid.data(
+                image_name = fluid.layers.data(
                     name='image_name',
-                    shape=[-1, self.cfg.batch_size],
+                    shape=[self.cfg.batch_size],
                     dtype="int32")
                 test_py_reader = fluid.io.PyReader(
                     feed_list=[input_A, input_B, input_C, image_name],

PaddleCV/PaddleGAN/trainer/STGAN.py

@@ -284,16 +284,16 @@ class STGAN(object):
     def build_model(self):
         data_shape = [-1, 3, self.cfg.image_size, self.cfg.image_size]
 
-        image_real = fluid.data(
+        image_real = fluid.layers.data(
             name='image_real', shape=data_shape, dtype='float32')
-        label_org = fluid.data(
-            name='label_org', shape=[-1, self.cfg.c_dim], dtype='float32')
-        label_trg = fluid.data(
-            name='label_trg', shape=[-1, self.cfg.c_dim], dtype='float32')
-        label_org_ = fluid.data(
-            name='label_org_', shape=[-1, self.cfg.c_dim], dtype='float32')
-        label_trg_ = fluid.data(
-            name='label_trg_', shape=[-1, self.cfg.c_dim], dtype='float32')
+        label_org = fluid.layers.data(
+            name='label_org', shape=[self.cfg.c_dim], dtype='float32')
+        label_trg = fluid.layers.data(
+            name='label_trg', shape=[self.cfg.c_dim], dtype='float32')
+        label_org_ = fluid.layers.data(
+            name='label_org_', shape=[self.cfg.c_dim], dtype='float32')
+        label_trg_ = fluid.layers.data(
+            name='label_trg_', shape=[self.cfg.c_dim], dtype='float32')
 
         test_gen_trainer = GTrainer(image_real, label_org, label_org_,
                                     label_trg, label_trg_, self.cfg,
@@ -378,9 +378,9 @@ class STGAN(object):
                 batch_id += 1
 
             if self.cfg.run_test:
-                image_name = fluid.data(
+                image_name = fluid.layers.data(
                     name='image_name',
-                    shape=[-1, self.cfg.n_samples],
+                    shape=[self.cfg.n_samples],
                     dtype='int32')
                 test_py_reader = fluid.io.PyReader(
                     feed_list=[image_real, label_org, label_trg, image_name],

PaddleCV/PaddleGAN/trainer/StarGAN.py

@@ -261,12 +261,12 @@ class StarGAN(object):
     def build_model(self):
         data_shape = [-1, 3, self.cfg.image_size, self.cfg.image_size]
 
-        image_real = fluid.data(
+        image_real = fluid.layers.data(
             name='image_real', shape=data_shape, dtype='float32')
-        label_org = fluid.data(
-            name='label_org', shape=[-1, self.cfg.c_dim], dtype='float32')
-        label_trg = fluid.data(
-            name='label_trg', shape=[-1, self.cfg.c_dim], dtype='float32')
+        label_org = fluid.layers.data(
+            name='label_org', shape=[self.cfg.c_dim], dtype='float32')
+        label_trg = fluid.layers.data(
+            name='label_trg', shape=[self.cfg.c_dim], dtype='float32')
 
         py_reader = fluid.io.PyReader(
             feed_list=[image_real, label_org, label_trg],
@@ -346,9 +346,9 @@ class StarGAN(object):
                 batch_id += 1
 
             if self.cfg.run_test:
-                image_name = fluid.data(
+                image_name = fluid.layers.data(
                     name='image_name',
-                    shape=[-1, self.cfg.n_samples],
+                    shape=[self.cfg.n_samples],
                     dtype='int32')
                 test_py_reader = fluid.io.PyReader(
                     feed_list=[image_real, label_org, label_trg, image_name],