change some model using data loader (#4595)

dygraph/mnist/train.py

@@ -99,7 +99,9 @@ class MNIST(fluid.dygraph.Layer):
         self.pool_2_shape = 50 * 4 * 4
         SIZE = 10
         scale = (2.0 / (self.pool_2_shape**2 * SIZE))**0.5
-        self._fc = Linear(self.pool_2_shape, 10,
+        self._fc = Linear(
+            self.pool_2_shape,
+            10,
             param_attr=fluid.param_attr.ParamAttr(
                 initializer=fluid.initializer.NormalInitializer(
                     loc=0.0, scale=scale)),
@@ -117,17 +119,21 @@ class MNIST(fluid.dygraph.Layer):
             return x
 
 
+def reader_decorator(reader):
+    def __reader__():
+        for item in reader():
+            img = np.array(item[0]).astype('float32').reshape(1, 28, 28)
+            label = np.array(item[1]).astype('int64').reshape(1)
+            yield img, label
+
+    return __reader__
+
+
 def test_mnist(reader, model, batch_size):
     acc_set = []
     avg_loss_set = []
     for batch_id, data in enumerate(reader()):
-        dy_x_data = np.array([x[0].reshape(1, 28, 28)
-                              for x in data]).astype('float32')
-        y_data = np.array(
-            [x[1] for x in data]).astype('int64').reshape(batch_size, 1)
-
-        img = to_variable(dy_x_data)
-        label = to_variable(y_data)
+        img, label = data
         label.stop_gradient = True
         prediction, acc = model(img, label)
         loss = fluid.layers.cross_entropy(input=prediction, label=label)
@@ -187,28 +193,33 @@ def train_mnist(args):
         if args.use_data_parallel:
             strategy = fluid.dygraph.parallel.prepare_context()
         mnist = MNIST()
-        adam = AdamOptimizer(learning_rate=0.001, parameter_list=mnist.parameters())
+        adam = AdamOptimizer(
+            learning_rate=0.001, parameter_list=mnist.parameters())
         if args.use_data_parallel:
             mnist = fluid.dygraph.parallel.DataParallel(mnist, strategy)
 
         train_reader = paddle.batch(
-            paddle.dataset.mnist.train(), batch_size=BATCH_SIZE, drop_last=True)
+            reader_decorator(paddle.dataset.mnist.train()),
+            batch_size=BATCH_SIZE,
+            drop_last=True)
         if args.use_data_parallel:
             train_reader = fluid.contrib.reader.distributed_batch_reader(
                 train_reader)
 
         test_reader = paddle.batch(
-            paddle.dataset.mnist.test(), batch_size=BATCH_SIZE, drop_last=True)
+            reader_decorator(paddle.dataset.mnist.test()),
+            batch_size=BATCH_SIZE,
+            drop_last=True)
+
+        train_loader = fluid.io.DataLoader.from_generator(capacity=10)
+        train_loader.set_sample_list_generator(train_reader, places=place)
+
+        test_loader = fluid.io.DataLoader.from_generator(capacity=10)
+        test_loader.set_sample_list_generator(test_reader, places=place)
 
         for epoch in range(epoch_num):
-            for batch_id, data in enumerate(train_reader()):
-                dy_x_data = np.array([x[0].reshape(1, 28, 28)
-                                      for x in data]).astype('float32')
-                y_data = np.array(
-                    [x[1] for x in data]).astype('int64').reshape(-1, 1)
-
-                img = to_variable(dy_x_data)
-                label = to_variable(y_data)
+            for batch_id, data in enumerate(train_loader()):
+                img, label = data
                 label.stop_gradient = True
 
                 cost, acc = mnist(img, label)
@@ -231,7 +242,7 @@ def train_mnist(args):
                         epoch, batch_id, avg_loss.numpy()))
 
             mnist.eval()
-            test_cost, test_acc = test_mnist(test_reader, mnist, BATCH_SIZE)
+            test_cost, test_acc = test_mnist(test_loader, mnist, BATCH_SIZE)
             mnist.train()
             if args.ce:
                 print("kpis\ttest_acc\t%s" % test_acc)

dygraph/mobilenet/reader.py

@@ -239,7 +239,7 @@ def process_image(sample, settings, mode, color_jitter, rotate):
     img /= img_std
 
     if mode == 'train' or mode == 'val':
-        return (img, sample[1])
+        return (img, [sample[1]])
     elif mode == 'test':
         return (img, )
 

dygraph/mobilenet/train.py

@@ -116,10 +116,8 @@ def train_mobilenet():
             optimizer.set_dict(opti_dict)
 
         # 3. reader
-        train_data_loader, train_data = utility.create_data_loader(
-            is_train=True, args=args)
-        test_data_loader, test_data = utility.create_data_loader(
-            is_train=False, args=args)
+        train_data_loader = utility.create_data_loader(is_train=True, args=args)
+        test_data_loader = utility.create_data_loader(is_train=False, args=args)
         num_trainers = int(os.environ.get('PADDLE_TRAINERS_NUM', 1))
         imagenet_reader = reader.ImageNetReader(seed=0, place_num=place_num)
         train_reader = imagenet_reader.train(settings=args)
@@ -145,8 +143,6 @@ def train_mobilenet():
                 t1 = time.time()
                 if args.max_iter and total_batch_num == args.max_iter:
                     return
-                label = to_variable(label.numpy().astype('int64').reshape(
-                    int(args.batch_size // place_num), 1))
                 t_start = time.time()
 
                 # 4.1.1 call net()

dygraph/mobilenet/utils/utility.py

@@ -309,32 +309,14 @@ def create_data_loader(is_train, args):
     Returns:
         data_loader and the input data of net, 
     """
-    image_shape = [int(m) for m in args.image_shape.split(",")]
-
-    feed_image = fluid.data(
-        name="feed_image",
-        shape=[None] + image_shape,
-        dtype="float32",
-        lod_level=0)
-
-    feed_label = fluid.data(
-        name="feed_label", shape=[None, 1], dtype="int64", lod_level=0)
-    feed_y_a = fluid.data(
-        name="feed_y_a", shape=[None, 1], dtype="int64", lod_level=0)
-
     if is_train and args.use_mixup:
-        feed_y_b = fluid.data(
-            name="feed_y_b", shape=[None, 1], dtype="int64", lod_level=0)
-        feed_lam = fluid.data(
-            name="feed_lam", shape=[None, 1], dtype="float32", lod_level=0)
-
         data_loader = fluid.io.DataLoader.from_generator(
             capacity=64,
             use_double_buffer=True,
             iterable=True,
             return_list=True)
 
-        return data_loader, [feed_image, feed_y_a, feed_y_b, feed_lam]
+        return data_loader
     else:
         data_loader = fluid.io.DataLoader.from_generator(
             capacity=64,
@@ -342,7 +324,7 @@ def create_data_loader(is_train, args):
             iterable=True,
             return_list=True)
 
-        return data_loader, [feed_image, feed_label]
+        return data_loader
 
 
 def print_info(pass_id, batch_id, print_step, metrics, time_info, info_mode):

dygraph/ptb_lm/ptb_dy.py

@@ -91,7 +91,7 @@ class SimpleLSTMRNN(fluid.Layer):
 
         res = []
         for index in range(self._num_steps):
-            step_input = input_embedding[:,index,:]
+            step_input = input_embedding[:, index, :]
             for k in range(self._num_layers):
                 pre_hidden = hidden_array[k]
                 pre_cell = cell_array[k]
@@ -118,7 +118,8 @@ class SimpleLSTMRNN(fluid.Layer):
                         dropout_implementation='upscale_in_train')
             res.append(step_input)
         real_res = fluid.layers.concat(res, 1)
-        real_res = fluid.layers.reshape(real_res, [ -1, self._num_steps, self._hidden_size])
+        real_res = fluid.layers.reshape(
+            real_res, [-1, self._num_steps, self._hidden_size])
         last_hidden = fluid.layers.concat(hidden_array, 1)
         last_hidden = fluid.layers.reshape(
             last_hidden, shape=[-1, self._num_layers, self._hidden_size])
@@ -334,10 +335,20 @@ def train_ptb_lm():
 
         grad_clip = fluid.clip.GradientClipByGlobalNorm(max_grad_norm)
         sgd = SGDOptimizer(
-            learning_rate=fluid.layers.piecewise_decay(boundaries=bd, values=lr_arr), 
+            learning_rate=fluid.layers.piecewise_decay(
+                boundaries=bd, values=lr_arr),
             parameter_list=ptb_model.parameters(),
             grad_clip=grad_clip)
 
+        def reader_decorator(reader):
+            def __reader__():
+                for item in reader:
+                    x_data = item[0].reshape((-1, num_steps, 1))
+                    y_data = item[1].reshape((-1, num_steps, 1))
+                    yield x_data, y_data
+
+            return __reader__
+
         def eval(model, data):
             print("begin to eval")
             total_loss = 0.0
@@ -348,13 +359,14 @@ def train_ptb_lm():
                 (num_layers, batch_size, hidden_size), dtype='float32')
 
             model.eval()
-            train_data_iter = reader.get_data_iter(data, batch_size, num_steps)
-            for batch_id, batch in enumerate(train_data_iter):
-                x_data, y_data = batch
-                x_data = x_data.reshape((-1, num_steps, 1))
-                y_data = y_data.reshape((-1, num_steps, 1))
-                x = to_variable(x_data)
-                y = to_variable(y_data)
+            train_data_iter = reader_decorator(
+                reader.get_data_iter(data, batch_size, num_steps))
+
+            eval_data_loader = fluid.io.DataLoader.from_generator(capacity=200)
+            eval_data_loader.set_batch_generator(train_data_iter, places=place)
+
+            for batch_id, batch in enumerate(eval_data_loader):
+                x, y = batch
                 init_hidden = to_variable(init_hidden_data)
                 init_cell = to_variable(init_cell_data)
                 dy_loss, last_hidden, last_cell = ptb_model(x, y, init_hidden,
@@ -385,22 +397,20 @@ def train_ptb_lm():
             init_cell_data = np.zeros(
                 (num_layers, batch_size, hidden_size), dtype='float32')
 
-            train_data_iter = reader.get_data_iter(train_data, batch_size,
-                                                   num_steps)
+            train_data_iter = reader_decorator(
+                reader.get_data_iter(train_data, batch_size, num_steps))
+
+            train_data_loader = fluid.io.DataLoader.from_generator(capacity=200)
+            train_data_loader.set_batch_generator(train_data_iter, places=place)
+
             init_hidden = to_variable(init_hidden_data)
             init_cell = to_variable(init_cell_data)
             start_time = time.time()
-            for batch_id, batch in enumerate(train_data_iter):
+            for batch_id, batch in enumerate(train_data_loader):
                 if args.max_iter and total_batch_num == args.max_iter:
                     return
                 batch_start = time.time()
-                x_data, y_data = batch
-
-                x_data = x_data.reshape((-1, num_steps, 1))
-                y_data = y_data.reshape((-1, num_steps, 1))
-
-                x = to_variable(x_data)
-                y = to_variable(y_data)
+                x, y = batch
 
                 dy_loss, last_hidden, last_cell = ptb_model(x, y, init_hidden,
                                                             init_cell)
@@ -434,7 +444,9 @@ def train_ptb_lm():
             if batch_size <= 20 and epoch_id == 0 and ppl[0] > 1000:
                 # for bad init, after first epoch, the loss is over 1000
                 # no more need to continue
-                print("Parameters are randomly initialized and not good this time because the loss is over 1000 after the first epoch.")
+                print(
+                    "Parameters are randomly initialized and not good this time because the loss is over 1000 after the first epoch."
+                )
                 print("Abort this training process and please start again.")
                 return
 
@@ -458,4 +470,5 @@ def train_ptb_lm():
 
         eval(ptb_model, test_data)
 
+
 train_ptb_lm()

dygraph/resnet/train.py

@@ -82,7 +82,6 @@ def optimizer_setting(parameter_list=None):
             momentum=momentum_rate,
             regularization=fluid.regularizer.L2Decay(l2_decay))
 
-
     return optimizer
 
 
@@ -116,11 +115,7 @@ class ConvBNLayer(fluid.dygraph.Layer):
 
 
 class BottleneckBlock(fluid.dygraph.Layer):
-    def __init__(self,
-                 num_channels,
-                 num_filters,
-                 stride,
-                 shortcut=True):
+    def __init__(self, num_channels, num_filters, stride, shortcut=True):
         super(BottleneckBlock, self).__init__()
 
         self.conv0 = ConvBNLayer(
@@ -186,16 +181,9 @@ class ResNet(fluid.dygraph.Layer):
         num_filters = [64, 128, 256, 512]
 
         self.conv = ConvBNLayer(
-            num_channels=3,
-            num_filters=64,
-            filter_size=7,
-            stride=2,
-            act='relu')
+            num_channels=3, num_filters=64, filter_size=7, stride=2, act='relu')
         self.pool2d_max = Pool2D(
-            pool_size=3,
-            pool_stride=2,
-            pool_padding=1,
-            pool_type='max')
+            pool_size=3, pool_stride=2, pool_padding=1, pool_type='max')
 
         self.bottleneck_block_list = []
         for block in range(len(depth)):
@@ -220,7 +208,8 @@ class ResNet(fluid.dygraph.Layer):
         import math
         stdv = 1.0 / math.sqrt(2048 * 1.0)
 
-        self.out = Linear(self.pool2d_avg_output,
+        self.out = Linear(
+            self.pool2d_avg_output,
             class_dim,
             act='softmax',
             param_attr=fluid.param_attr.ParamAttr(
@@ -237,6 +226,16 @@ class ResNet(fluid.dygraph.Layer):
         return y
 
 
+def reader_decorator(reader):
+    def __reader__():
+        for item in reader():
+            img = np.array(item[0]).astype('float32').reshape(3, 224, 224)
+            label = np.array(item[1]).astype('int64').reshape(1)
+            yield img, label
+
+    return __reader__
+
+
 def eval(model, data):
 
     model.eval()
@@ -245,15 +244,8 @@ def eval(model, data):
     total_acc5 = 0.0
     total_sample = 0
     for batch_id, data in enumerate(data()):
-        dy_x_data = np.array(
-            [x[0].reshape(3, 224, 224) for x in data]).astype('float32')
-        if len(np.array([x[1] for x in data]).astype('int64')) != batch_size:
-            continue
-        y_data = np.array([x[1] for x in data]).astype('int64').reshape(
-            batch_size, 1)
-
-        img = to_variable(dy_x_data)
-        label = to_variable(y_data)
+        img = data[0]
+        label = data[1]
         label.stop_gradient = True
 
         out = model(img)
@@ -303,13 +295,24 @@ def train_resnet():
             resnet = fluid.dygraph.parallel.DataParallel(resnet, strategy)
 
         train_reader = paddle.batch(
-            paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size)
+            reader_decorator(paddle.dataset.flowers.train(use_xmap=True)),
+            batch_size=batch_size,
+            drop_last=True)
+
         if args.use_data_parallel:
             train_reader = fluid.contrib.reader.distributed_batch_reader(
                 train_reader)
 
         test_reader = paddle.batch(
-            paddle.dataset.flowers.test(use_xmap=False), batch_size=batch_size)
+            reader_decorator(paddle.dataset.flowers.test(use_xmap=True)),
+            batch_size=batch_size,
+            drop_last=True)
+
+        train_loader = fluid.io.DataLoader.from_generator(capacity=10)
+        train_loader.set_sample_list_generator(train_reader, places=place)
+
+        test_loader = fluid.io.DataLoader.from_generator(capacity=10)
+        test_loader.set_sample_list_generator(test_reader, places=place)
 
         #file_name = './model/epoch_0.npz'
         #model_data = np.load( file_name )
@@ -331,23 +334,13 @@ def train_resnet():
 
             print("load finished")
 
-            for batch_id, data in enumerate(train_reader()):
-
+            for batch_id, data in enumerate(train_loader()):
                 #NOTE: used in benchmark
                 if args.max_iter and total_batch_num == args.max_iter:
                     return
                 batch_start = time.time()
 
-                dy_x_data = np.array(
-                    [x[0].reshape(3, 224, 224) for x in data]).astype('float32')
-                if len(np.array([x[1]
-                                 for x in data]).astype('int64')) != batch_size:
-                    continue
-                y_data = np.array([x[1] for x in data]).astype('int64').reshape(
-                    -1, 1)
-
-                img = to_variable(dy_x_data)
-                label = to_variable(y_data)
+                img, label = data
                 label.stop_gradient = True
 
                 out = resnet(img)
@@ -390,16 +383,14 @@ def train_resnet():
                   (eop, batch_id, total_loss / total_sample, \
                    total_acc1 / total_sample, total_acc5 / total_sample))
             resnet.eval()
-            eval(resnet, test_reader)
+            eval(resnet, test_loader)
 
             save_parameters = (not args.use_data_parallel) or (
                 args.use_data_parallel and
                 fluid.dygraph.parallel.Env().local_rank == 0)
             if save_parameters:
-                fluid.save_dygraph(resnet.state_dict(),
-                                                'resnet_params')
+                fluid.save_dygraph(resnet.state_dict(), 'resnet_params')
 
 
 if __name__ == '__main__':
-
     train_resnet()

dygraph/se_resnet/train.py

@@ -169,8 +169,7 @@ class BottleneckBlock(fluid.dygraph.Layer):
             act=None)
 
         self.scale = SqueezeExcitation(
-            num_channels=num_filters * 2,
-            reduction_ratio=reduction_ratio)
+            num_channels=num_filters * 2, reduction_ratio=reduction_ratio)
 
         if not shortcut:
             self.short = ConvBNLayer(
@@ -219,10 +218,7 @@ class SeResNeXt(fluid.dygraph.Layer):
                 stride=2,
                 act='relu')
             self.pool = Pool2D(
-                pool_size=3,
-                pool_stride=2,
-                pool_padding=1,
-                pool_type='max')
+                pool_size=3, pool_stride=2, pool_padding=1, pool_type='max')
         elif layers == 101:
             cardinality = 32
             reduction_ratio = 16
@@ -235,10 +231,7 @@ class SeResNeXt(fluid.dygraph.Layer):
                 stride=2,
                 act='relu')
             self.pool = Pool2D(
-                pool_size=3,
-                pool_stride=2,
-                pool_padding=1,
-                pool_type='max')
+                pool_size=3, pool_stride=2, pool_padding=1, pool_type='max')
         elif layers == 152:
             cardinality = 64
             reduction_ratio = 16
@@ -263,10 +256,7 @@ class SeResNeXt(fluid.dygraph.Layer):
                 stride=1,
                 act='relu')
             self.pool = Pool2D(
-                pool_size=3,
-                pool_stride=2,
-                pool_padding=1,
-                pool_type='max')
+                pool_size=3, pool_stride=2, pool_padding=1, pool_type='max')
 
         self.bottleneck_block_list = []
         num_channels = 64
@@ -294,7 +284,8 @@ class SeResNeXt(fluid.dygraph.Layer):
 
         self.pool2d_avg_output = num_filters[len(num_filters) - 1] * 2 * 1 * 1
 
-        self.out = Linear(self.pool2d_avg_output,
+        self.out = Linear(
+            self.pool2d_avg_output,
             class_dim,
             param_attr=fluid.param_attr.ParamAttr(
                 initializer=fluid.initializer.Uniform(-stdv, stdv)))
@@ -318,6 +309,16 @@ class SeResNeXt(fluid.dygraph.Layer):
         return y
 
 
+def reader_decorator(reader):
+    def __reader__():
+        for item in reader():
+            img = np.array(item[0]).astype('float32').reshape(3, 224, 224)
+            label = np.array(item[1]).astype('int64').reshape(1)
+            yield img, label
+
+    return __reader__
+
+
 def eval(model, data):
 
     model.eval()
@@ -327,15 +328,7 @@ def eval(model, data):
     total_acc5 = 0.0
     total_sample = 0
     for batch_id, data in enumerate(data()):
-        dy_x_data = np.array(
-            [x[0].reshape(3, 224, 224) for x in data]).astype('float32')
-        if len(np.array([x[1] for x in data]).astype('int64')) != batch_size:
-            continue
-        y_data = np.array([x[1] for x in data]).astype('int64').reshape(
-            batch_size, 1)
-
-        img = to_variable(dy_x_data)
-        label = to_variable(y_data)
+        img, label = data
         label.stop_gradient = True
         out = model(img)
 
@@ -389,29 +382,29 @@ def train():
             se_resnext = fluid.dygraph.parallel.DataParallel(se_resnext,
                                                              strategy)
         train_reader = paddle.batch(
-            paddle.dataset.flowers.train(use_xmap=False),
+            reader_decorator(paddle.dataset.flowers.train(use_xmap=False)),
             batch_size=batch_size,
             drop_last=True)
         if args.use_data_parallel:
             train_reader = fluid.contrib.reader.distributed_batch_reader(
                 train_reader)
         test_reader = paddle.batch(
-            paddle.dataset.flowers.test(use_xmap=False), batch_size=32)
+            reader_decorator(paddle.dataset.flowers.test(use_xmap=False)),
+            batch_size=32)
+
+        train_loader = fluid.io.DataLoader.from_generator(capacity=10)
+        train_loader.set_sample_list_generator(train_reader, places=place)
+
+        test_loader = fluid.io.DataLoader.from_generator(capacity=10)
+        test_loader.set_sample_list_generator(test_reader, places=place)
 
         for epoch_id in range(epoch_num):
             total_loss = 0.0
             total_acc1 = 0.0
             total_acc5 = 0.0
             total_sample = 0
-            for batch_id, data in enumerate(train_reader()):
-
-                dy_x_data = np.array([x[0].reshape(3, 224, 224)
-                                      for x in data]).astype('float32')
-                y_data = np.array([x[1] for x in data]).astype('int64').reshape(
-                    batch_size, 1)
-
-                img = to_variable(dy_x_data)
-                label = to_variable(y_data)
+            for batch_id, data in enumerate(train_loader()):
+                img, label = data
                 label.stop_gradient = True
 
                 out = se_resnext(img)
@@ -454,7 +447,7 @@ def train():
                   (epoch_id, batch_id, total_loss / total_sample, \
                    total_acc1 / total_sample, total_acc5 / total_sample))
             se_resnext.eval()
-            eval(se_resnext, test_reader)
+            eval(se_resnext, test_loader)
             se_resnext.train()