fix fluid.data and fluid.io.DataLoader (#3533)

* fix fluid.data and fluid.io.DataLoader

* add logger in utils/utility.py

PaddleCV/PaddleVideo/README.md

@@ -25,7 +25,7 @@
 
 ## 安装
 
-在当前模型库运行样例代码需要PaddlePaddle Fluid v.1.5.0或以上的版本。如果你的运行环境中的PaddlePaddle低于此版本，请根据[安装文档](http://www.paddlepaddle.org/documentation/docs/zh/1.5/beginners_guide/install/index_cn.html)中的说明来更新PaddlePaddle。
+在当前模型库运行样例代码需要PaddlePaddle Fluid v.1.6.0或以上的版本。如果你的运行环境中的PaddlePaddle低于此版本，请根据[安装文档](http://www.paddlepaddle.org/documentation/docs/zh/1.6/beginners_guide/install/index_cn.html)中的说明来更新PaddlePaddle。
 
 ### 其他环境依赖
 

PaddleCV/PaddleVideo/eval.py

@@ -26,6 +26,7 @@ import models
 from reader import get_reader
 from metrics import get_metrics
 from utils.utility import check_cuda
+from utils.utility import check_version
 
 logging.root.handlers = []
 FORMAT = '[%(levelname)s: %(filename)s: %(lineno)4d]: %(message)s'
@@ -83,7 +84,7 @@ def test(args):
 
     # build model
     test_model = models.get_model(args.model_name, test_config, mode='test')
-    test_model.build_input(use_pyreader=False)
+    test_model.build_input(use_dataloader=False)
     test_model.build_model()
     test_feeds = test_model.feeds()
     test_fetch_list = test_model.fetches()
@@ -130,6 +131,7 @@ if __name__ == "__main__":
     args = parse_args()
     # check whether the installed paddle is compiled with GPU
     check_cuda(args.use_gpu)
+    check_version()
     logger.info(args)
 
     test(args)

PaddleCV/PaddleVideo/inference_model.py

@@ -80,7 +80,7 @@ def save_inference_model(args):
     infer_config = merge_configs(config, 'infer', vars(args))
     print_configs(infer_config, "Infer")
     infer_model = models.get_model(args.model_name, infer_config, mode='infer')
-    infer_model.build_input(use_pyreader=False)
+    infer_model.build_input(use_dataloader=False)
     infer_model.build_model()
     infer_feeds = infer_model.feeds()
     infer_outputs = infer_model.outputs()

PaddleCV/PaddleVideo/models/attention_cluster/attention_cluster.py

@@ -41,21 +41,21 @@ class AttentionCluster(ModelBase):
             self.learning_rate = self.get_config_from_sec('train',
                                                           'learning_rate', 1e-3)
 
-    def build_input(self, use_pyreader=True):
+    def build_input(self, use_dataloader=True):
         self.feature_input = []
         for name, dim in zip(self.feature_names, self.feature_dims):
             self.feature_input.append(
-                fluid.layers.data(
-                    shape=[self.seg_num, dim], dtype='float32', name=name))
+                fluid.data(
+                    shape=[None, self.seg_num, dim], dtype='float32', name=name))
         if self.mode != 'infer':
-            self.label_input = fluid.layers.data(
-                shape=[self.class_num], dtype='float32', name='label')
+            self.label_input = fluid.data(
+                shape=[None, self.class_num], dtype='float32', name='label')
         else:
             self.label_input = None
-        if use_pyreader:
+        if use_dataloader:
             assert self.mode != 'infer', \
-                    'pyreader is not recommendated when infer, please set use_pyreader to be false.'
-            self.py_reader = fluid.io.PyReader(
+                    'dataloader is not recommendated when infer, please set use_dataloader to be false.'
+            self.dataloader = fluid.io.DataLoader.from_generator(
                 feed_list=self.feature_input + [self.label_input],
                 capacity=8,
                 iterable=True)

PaddleCV/PaddleVideo/models/attention_lstm/attention_lstm.py

@@ -52,21 +52,21 @@ class AttentionLSTM(ModelBase):
             self.decay_gamma = self.get_config_from_sec('train', 'decay_gamma',
                                                         0.1)
 
-    def build_input(self, use_pyreader):
+    def build_input(self, use_dataloader):
         self.feature_input = []
         for name, dim in zip(self.feature_names, self.feature_dims):
             self.feature_input.append(
-                fluid.layers.data(
-                    shape=[dim], lod_level=1, dtype='float32', name=name))
+                fluid.data(
+                    shape=[None, dim], lod_level=1, dtype='float32', name=name))
         if self.mode != 'infer':
-            self.label_input = fluid.layers.data(
-                shape=[self.num_classes], dtype='float32', name='label')
+            self.label_input = fluid.data(
+                shape=[None, self.num_classes], dtype='float32', name='label')
         else:
             self.label_input = None
-        if use_pyreader:
+        if use_dataloader:
             assert self.mode != 'infer', \
-                    'pyreader is not recommendated when infer, please set use_pyreader to be false.'
-            self.py_reader = fluid.io.PyReader(
+                    'dataloader is not recommendated when infer, please set use_dataloader to be false.'
+            self.dataloader = fluid.io.DataLoader.from_generator(
                 feed_list=self.feature_input + [self.label_input],
                 capacity=8,
                 iterable=True)

PaddleCV/PaddleVideo/models/bmn/bmn.py

@@ -52,47 +52,46 @@ class BMN(ModelBase):
                                                         'l2_weight_decay')
         self.lr_decay_iter = self.get_config_from_sec('train', 'lr_decay_iter')
 
-    def build_input(self, use_pyreader=True):
-        feat_shape = [self.feat_dim, self.tscale]
-        gt_iou_map_shape = [self.dscale, self.tscale]
-        gt_start_shape = [self.tscale]
-        gt_end_shape = [self.tscale]
-        fileid_shape = [1]
-        self.use_pyreader = use_pyreader
+    def build_input(self, use_dataloader=True):
+        feat_shape = [None, self.feat_dim, self.tscale]
+        gt_iou_map_shape = [None, self.dscale, self.tscale]
+        gt_start_shape = [None, self.tscale]
+        gt_end_shape = [None, self.tscale]
+        fileid_shape = [None, 1]
+        self.use_dataloader = use_dataloader
         # set init data to None
-        py_reader = None
         feat = None
         gt_iou_map = None
         gt_start = None
         gt_end = None
         fileid = None
 
-        feat = fluid.layers.data(name='feat', shape=feat_shape, dtype='float32')
+        feat = fluid.data(name='feat', shape=feat_shape, dtype='float32')
 
         feed_list = []
         feed_list.append(feat)
         if (self.mode == 'train') or (self.mode == 'valid'):
-            gt_start = fluid.layers.data(
+            gt_start = fluid.data(
                 name='gt_start', shape=gt_start_shape, dtype='float32')
-            gt_end = fluid.layers.data(
+            gt_end = fluid.data(
                 name='gt_end', shape=gt_end_shape, dtype='float32')
-            gt_iou_map = fluid.layers.data(
+            gt_iou_map = fluid.data(
                 name='gt_iou_map', shape=gt_iou_map_shape, dtype='float32')
             feed_list.append(gt_iou_map)
             feed_list.append(gt_start)
             feed_list.append(gt_end)
 
         elif self.mode == 'test':
-            gt_start = fluid.layers.data(
+            gt_start = fluid.data(
                 name='gt_start', shape=gt_start_shape, dtype='float32')
-            gt_end = fluid.layers.data(
+            gt_end = fluid.data(
                 name='gt_end', shape=gt_end_shape, dtype='float32')
-            gt_iou_map = fluid.layers.data(
+            gt_iou_map = fluid.data(
                 name='gt_iou_map', shape=gt_iou_map_shape, dtype='float32')
             feed_list.append(gt_iou_map)
             feed_list.append(gt_start)
             feed_list.append(gt_end)
-            fileid = fluid.layers.data(
+            fileid = fluid.data(
                 name='fileid', shape=fileid_shape, dtype='int64')
             feed_list.append(fileid)
         elif self.mode == 'infer':
@@ -102,11 +101,11 @@ class BMN(ModelBase):
             raise NotImplementedError('mode {} not implemented'.format(
                 self.mode))
 
-        if use_pyreader:
+        if use_dataloader:
             assert self.mode != 'infer', \
-                        'pyreader is not recommendated when infer, please set use_pyreader to be false.'
-            self.py_reader = fluid.io.PyReader(
-                feed_list=feed_list, capacity=8, iterable=True)
+                        'dataloader is not recommendated when infer, please set use_dataloader to be false.'
+            self.dataloader = fluid.io.DataLoader.from_generator(
+                               feed_list=feed_list, capacity=8, iterable=True)
 
         self.feat_input = [feat]
         self.gt_iou_map = gt_iou_map

PaddleCV/PaddleVideo/models/bsn/bsn.py

@@ -47,47 +47,46 @@ class BsnTem(ModelBase):
                                                         'l2_weight_decay')
         self.lr_decay_iter = self.get_config_from_sec('train', 'lr_decay_iter')
 
-    def build_input(self, use_pyreader=True):
-        feat_shape = [self.feat_dim, self.tscale]
-        gt_start_shape = [self.tscale]
-        gt_end_shape = [self.tscale]
-        gt_action_shape = [self.tscale]
-        fileid_shape = [1]
-        self.use_pyreader = use_pyreader
+    def build_input(self, use_dataloader=True):
+        feat_shape = [None, self.feat_dim, self.tscale]
+        gt_start_shape = [None, self.tscale]
+        gt_end_shape = [None, self.tscale]
+        gt_action_shape = [None, self.tscale]
+        fileid_shape = [None, 1]
+        self.use_dataloader = use_dataloader
         # set init data to None
-        py_reader = None
         feat = None
         gt_start = None
         gt_end = None
         gt_action = None
         fileid = None
 
-        feat = fluid.layers.data(name='feat', shape=feat_shape, dtype='float32')
+        feat = fluid.data(name='feat', shape=feat_shape, dtype='float32')
 
         feed_list = []
         feed_list.append(feat)
         if (self.mode == 'train') or (self.mode == 'valid'):
-            gt_start = fluid.layers.data(
+            gt_start = fluid.data(
                 name='gt_start', shape=gt_start_shape, dtype='float32')
-            gt_end = fluid.layers.data(
+            gt_end = fluid.data(
                 name='gt_end', shape=gt_end_shape, dtype='float32')
-            gt_action = fluid.layers.data(
+            gt_action = fluid.data(
                 name='gt_action', shape=gt_action_shape, dtype='float32')
             feed_list.append(gt_start)
             feed_list.append(gt_end)
             feed_list.append(gt_action)
 
         elif self.mode == 'test':
-            gt_start = fluid.layers.data(
+            gt_start = fluid.data(
                 name='gt_start', shape=gt_start_shape, dtype='float32')
-            gt_end = fluid.layers.data(
+            gt_end = fluid.data(
                 name='gt_end', shape=gt_end_shape, dtype='float32')
-            gt_action = fluid.layers.data(
+            gt_action = fluid.data(
                 name='gt_action', shape=gt_action_shape, dtype='float32')
             feed_list.append(gt_start)
             feed_list.append(gt_end)
             feed_list.append(gt_action)
-            fileid = fluid.layers.data(
+            fileid = fluid.data(
                 name='fileid', shape=fileid_shape, dtype='int64')
             feed_list.append(fileid)
         elif self.mode == 'infer':
@@ -97,11 +96,11 @@ class BsnTem(ModelBase):
             raise NotImplementedError('mode {} not implemented'.format(
                 self.mode))
 
-        if use_pyreader:
+        if use_dataloader:
             assert self.mode != 'infer', \
-                        'pyreader is not recommendated when infer, please set use_pyreader to be false.'
-            self.py_reader = fluid.io.PyReader(
-                feed_list=feed_list, capacity=8, iterable=True)
+                        'dataloader is not recommendated when infer, please set use_dataloader to be false.'
+            self.dataloader = fluid.io.DataLoader.from_generator(
+                              feed_list=feed_list, capacity=8, iterable=True)
 
         self.feat_input = [feat]
         self.gt_start = gt_start
@@ -212,51 +211,50 @@ class BsnPem(ModelBase):
                                                         'l2_weight_decay')
         self.lr_decay_iter = self.get_config_from_sec('train', 'lr_decay_iter')
 
-    def build_input(self, use_pyreader=True):
-        feat_shape = [self.top_K, self.feat_dim]
-        gt_iou_shape = [self.top_K, 1]
-        props_info_shape = [self.top_K, 4]
-        fileid_shape = [1]
-        self.use_pyreader = use_pyreader
+    def build_input(self, use_dataloader=True):
+        feat_shape = [None, self.top_K, self.feat_dim]
+        gt_iou_shape = [None. self.top_K, 1]
+        props_info_shape = [None, self.top_K, 4]
+        fileid_shape = [None, 1]
+        self.use_dataloader = use_dataloader
         # set init data to None
-        py_reader = None
         feat = None
         gt_iou = None
         props_info = None
         fileid = None
 
-        feat = fluid.layers.data(name='feat', shape=feat_shape, dtype='float32')
+        feat = fluid.data(name='feat', shape=feat_shape, dtype='float32')
 
         feed_list = []
         feed_list.append(feat)
         if (self.mode == 'train') or (self.mode == 'valid'):
-            gt_iou = fluid.layers.data(
+            gt_iou = fluid.data(
                 name='gt_iou', shape=gt_iou_shape, dtype='float32')
             feed_list.append(gt_iou)
 
         elif self.mode == 'test':
-            gt_iou = fluid.layers.data(
+            gt_iou = fluid.data(
                 name='gt_iou', shape=gt_iou_shape, dtype='float32')
-            props_info = fluid.layers.data(
+            props_info = fluid.data(
                 name='props_info', shape=props_info_shape, dtype='float32')
             feed_list.append(gt_iou)
             feed_list.append(props_info)
-            fileid = fluid.layers.data(
+            fileid = fluid.data(
                 name='fileid', shape=fileid_shape, dtype='int64')
             feed_list.append(fileid)
         elif self.mode == 'infer':
-            props_info = fluid.layers.data(
+            props_info = fluid.data(
                 name='props_info', shape=props_info_shape, dtype='float32')
             feed_list.append(props_info)
         else:
             raise NotImplementedError('mode {} not implemented'.format(
                 self.mode))
 
-        if use_pyreader:
+        if use_dataloader:
             assert self.mode != 'infer', \
-                        'pyreader is not recommendated when infer, please set use_pyreader to be false.'
-            self.py_reader = fluid.io.PyReader(
-                feed_list=feed_list, capacity=4, iterable=True)
+                        'dataloader is not recommendated when infer, please set use_dataloader to be false.'
+            self.dataloader = fluid.io.DataLoader.from_generator(
+                                feed_list=feed_list, capacity=4, iterable=True)
 
         self.feat_input = [feat]
         self.gt_iou = gt_iou

PaddleCV/PaddleVideo/models/ctcn/ctcn.py

@@ -51,37 +51,36 @@ class CTCN(ModelBase):
         self.momentum = self.get_config_from_sec('train', 'momentum')
         self.lr_decay_iter = self.get_config_from_sec('train', 'lr_decay_iter')
 
-    def build_input(self, use_pyreader=True):
-        image_shape = [1, self.img_size, self.concept_size]
-        loc_shape = [self.total_num_anchors, 2]
-        cls_shape = [self.total_num_anchors]
-        fileid_shape = [1]
-        self.use_pyreader = use_pyreader
+    def build_input(self, use_dataloader=True):
+        image_shape = [None, 1, self.img_size, self.concept_size]
+        loc_shape = [None, self.total_num_anchors, 2]
+        cls_shape = [None, self.total_num_anchors]
+        fileid_shape = [None, 1]
+        self.use_dataloader = use_dataloader
         # set init data to None
-        py_reader = None
         image = None
         loc_targets = None
         cls_targets = None
         fileid = None
 
-        image = fluid.layers.data(
+        image = fluid.data(
             name='image', shape=image_shape, dtype='float32')
 
         feed_list = []
         feed_list.append(image)
         if (self.mode == 'train') or (self.mode == 'valid'):
-            loc_targets = fluid.layers.data(
+            loc_targets = fluid.data(
                 name='loc_targets', shape=loc_shape, dtype='float32')
-            cls_targets = fluid.layers.data(
+            cls_targets = fluid.data(
                 name='cls_targets', shape=cls_shape, dtype='int64')
             feed_list.append(loc_targets)
             feed_list.append(cls_targets)
         elif self.mode == 'test':
-            loc_targets = fluid.layers.data(
+            loc_targets = fluid.data(
                 name='loc_targets', shape=loc_shape, dtype='float32')
-            cls_targets = fluid.layers.data(
+            cls_targets = fluid.data(
                 name='cls_targets', shape=cls_shape, dtype='int64')
-            fileid = fluid.layers.data(
+            fileid = fluid.data(
                 name='fileid', shape=fileid_shape, dtype='int64')
             feed_list.append(loc_targets)
             feed_list.append(cls_targets)
@@ -93,11 +92,11 @@ class CTCN(ModelBase):
             raise NotImplementedError('mode {} not implemented'.format(
                 self.mode))
 
-        if use_pyreader:
+        if use_dataloader:
             assert self.mode != 'infer', \
-                        'pyreader is not recommendated when infer, please set use_pyreader to be false.'
-            self.py_reader = fluid.io.PyReader(
-                feed_list=feed_list, capacity=4, iterable=True)
+                        'dataloader is not recommendated when infer, please set use_dataloader to be false.'
+            self.dataloader = fluid.io.DataLoader.from_generator(
+                               feed_list=feed_list, capacity=4, iterable=True)
 
         self.feature_input = [image]
         self.cls_targets = cls_targets

PaddleCV/PaddleVideo/models/model.py

@@ -65,13 +65,13 @@ class ModelBase(object):
         self.is_training = (mode == 'train')
         self.mode = mode
         self.cfg = cfg
-        self.py_reader = None
+        self.dataloader = None
 
     def build_model(self):
         "build model struct"
         raise NotImplementError(self, self.build_model)
 
-    def build_input(self, use_pyreader):
+    def build_input(self, use_dataloader):
         "build input Variable"
         raise NotImplementError(self, self.build_input)
 
@@ -114,8 +114,8 @@ class ModelBase(object):
         wget.download(url, path)
         return path
 
-    def pyreader(self):
-        return self.py_reader
+    def dataloader(self):
+        return self.dataloader
 
     def epoch_num(self):
         "get train epoch num"

PaddleCV/PaddleVideo/models/nextvlad/nextvlad.py

@@ -61,17 +61,17 @@ class NEXTVLAD(ModelBase):
         # other params
         self.batch_size = self.get_config_from_sec(self.mode, 'batch_size')
 
-    def build_input(self, use_pyreader=True):
-        rgb_shape = [self.video_feature_size]
-        audio_shape = [self.audio_feature_size]
-        label_shape = [self.num_classes]
+    def build_input(self, use_dataloader=True):
+        rgb_shape = [None, self.video_feature_size]
+        audio_shape = [None, self.audio_feature_size]
+        label_shape = [None, self.num_classes]
 
-        rgb = fluid.layers.data(
+        rgb = fluid.data(
             name='train_rgb' if self.is_training else 'test_rgb',
             shape=rgb_shape,
             dtype='uint8',
             lod_level=1)
-        audio = fluid.layers.data(
+        audio = fluid.data(
             name='train_audio' if self.is_training else 'test_audio',
             shape=audio_shape,
             dtype='uint8',
@@ -79,16 +79,16 @@ class NEXTVLAD(ModelBase):
         if self.mode == 'infer':
             label = None
         else:
-            label = fluid.layers.data(
+            label = fluid.data(
                 name='train_label' if self.is_training else 'test_label',
                 shape=label_shape,
                 dtype='float32')
 
-        if use_pyreader:
+        if use_dataloader:
             assert self.mode != 'infer', \
-                    'pyreader is not recommendated when infer, please set use_pyreader to be false.'
-            self.py_reader = fluid.io.PyReader(
-                feed_list=[rgb, audio, label], capacity=8, iterable=True)
+                    'dataloader is not recommendated when infer, please set use_dataloader to be false.'
+            self.dataloader = fluid.io.DataLoader.from_generator(
+                                feed_list=[rgb, audio, label], capacity=8, iterable=True)
         self.feature_input = [rgb, audio]
         self.label_input = label
 

PaddleCV/PaddleVideo/models/nonlocal_model/nonlocal_model.py

@@ -39,27 +39,27 @@ class NonLocal(ModelBase):
         # crop size
         self.crop_size = self.get_config_from_sec(self.mode, 'crop_size')
 
-    def build_input(self, use_pyreader=True):
-        input_shape = [3, self.video_length, self.crop_size, self.crop_size]
-        label_shape = [1]
+    def build_input(self, use_dataloader=True):
+        input_shape = [None, 3, self.video_length, self.crop_size, self.crop_size]
+        label_shape = [None, 1]
 
-        data = fluid.layers.data(
+        data = fluid.data(
             name='train_data' if self.is_training else 'test_data',
             shape=input_shape,
             dtype='float32')
         if self.mode != 'infer':
-            label = fluid.layers.data(
+            label = fluid.data(
                 name='train_label' if self.is_training else 'test_label',
                 shape=label_shape,
                 dtype='int64')
         else:
             label = None
 
-        if use_pyreader:
+        if use_dataloader:
             assert self.mode != 'infer', \
-                        'pyreader is not recommendated when infer, please set use_pyreader to be false.'
-            self.py_reader = fluid.io.PyReader(
-                feed_list=[data, label], capacity=4, iterable=True)
+                        'dataloader is not recommendated when infer, please set use_dataloader to be false.'
+            self.dataloader = fluid.io.DataLoader.from_generator(
+                               feed_list=[data, label], capacity=4, iterable=True)
 
         self.feature_input = [data]
         self.label_input = label

PaddleCV/PaddleVideo/models/stnet/stnet.py

@@ -51,25 +51,24 @@ class STNET(ModelBase):
         self.target_size = self.get_config_from_sec(self.mode, 'target_size')
         self.batch_size = self.get_config_from_sec(self.mode, 'batch_size')
 
-    def build_input(self, use_pyreader=True):
+    def build_input(self, use_dataloader=True):
         image_shape = [3, self.target_size, self.target_size]
         image_shape[0] = image_shape[0] * self.seglen
-        image_shape = [self.seg_num] + image_shape
-        self.use_pyreader = use_pyreader
+        image_shape = [None, self.seg_num] + image_shape
+        self.use_dataloader = use_dataloader
 
-        image = fluid.layers.data(
+        image = fluid.data(
             name='image', shape=image_shape, dtype='float32')
         if self.mode != 'infer':
-            label = fluid.layers.data(name='label', shape=[1], dtype='int64')
+            label = fluid.data(name='label', shape=[None, 1], dtype='int64')
         else:
             label = None
 
-        if use_pyreader:
+        if use_dataloader:
             assert self.mode != 'infer', \
-                        'pyreader is not recommendated when infer, please set use_pyreader to be false.'
-            py_reader = fluid.io.PyReader(
-                feed_list=[image, label], capacity=4, iterable=True)
-            self.py_reader = py_reader
+                        'dataloader is not recommendated when infer, please set use_dataloader to be false.'
+            self.dataloader = fluid.io.DataLoader.from_generator(
+                            feed_list=[image, label], capacity=4, iterable=True)
 
         self.feature_input = [image]
         self.label_input = label
@@ -167,3 +166,4 @@ class STNET(ModelBase):
         param_numpy = np.mean(param_numpy, axis=1, keepdims=True) / self.seglen
         param_numpy = np.repeat(param_numpy, 3 * self.seglen, axis=1)
         param_tensor.set(param_numpy.astype(np.float32), place)
+

PaddleCV/PaddleVideo/models/tsm/tsm.py

@@ -51,25 +51,24 @@ class TSM(ModelBase):
         self.target_size = self.get_config_from_sec(self.mode, 'target_size')
         self.batch_size = self.get_config_from_sec(self.mode, 'batch_size')
 
-    def build_input(self, use_pyreader=True):
+    def build_input(self, use_dataloader=True):
         image_shape = [3, self.target_size, self.target_size]
         image_shape[0] = image_shape[0] * self.seglen
-        image_shape = [self.seg_num] + image_shape
-        self.use_pyreader = use_pyreader
+        image_shape = [None, self.seg_num] + image_shape
+        self.use_dataloader = use_dataloader
 
-        image = fluid.layers.data(
+        image = fluid.data(
             name='image', shape=image_shape, dtype='float32')
         if self.mode != 'infer':
-            label = fluid.layers.data(name='label', shape=[1], dtype='int64')
+            label = fluid.data(name='label', shape=[None, 1], dtype='int64')
         else:
             label = None
 
-        if use_pyreader:
+        if use_dataloader:
             assert self.mode != 'infer', \
-                        'pyreader is not recommendated when infer, please set use_pyreader to be false.'
-            py_reader = fluid.io.PyReader(
-                feed_list=[image, label], capacity=4, iterable=True)
-            self.py_reader = py_reader
+                        'dataloader is not recommendated when infer, please set use_dataloader to be false.'
+            self.dataloader = fluid.io.DataLoader.from_generator(
+                                feed_list=[image, label], capacity=4, iterable=True)
 
         self.feature_input = [image]
         self.label_input = label

PaddleCV/PaddleVideo/models/tsn/tsn.py

@@ -52,25 +52,24 @@ class TSN(ModelBase):
         self.target_size = self.get_config_from_sec(self.mode, 'target_size')
         self.batch_size = self.get_config_from_sec(self.mode, 'batch_size')
 
-    def build_input(self, use_pyreader=True):
+    def build_input(self, use_dataloader=True):
         image_shape = [3, self.target_size, self.target_size]
         image_shape[0] = image_shape[0] * self.seglen
-        image_shape = [self.seg_num] + image_shape
-        self.use_pyreader = use_pyreader
+        image_shape = [None, self.seg_num] + image_shape
+        self.use_dataloader = use_dataloader
 
-        image = fluid.layers.data(
+        image = fluid.data(
             name='image', shape=image_shape, dtype='float32')
         if self.mode != 'infer':
-            label = fluid.layers.data(name='label', shape=[1], dtype='int64')
+            label = fluid.data(name='label', shape=[None, 1], dtype='int64')
         else:
             label = None
 
-        if use_pyreader:
+        if use_dataloader:
             assert self.mode != 'infer', \
-                        'pyreader is not recommendated when infer, please set use_pyreader to be false.'
-            py_reader = fluid.io.PyReader(
-                feed_list=[image, label], capacity=4, iterable=True)
-            self.py_reader = py_reader
+                        'dataloader is not recommendated when infer, please set use_dataloader to be false.'
+            self.dataloader = fluid.io.DataLoader.from_generator(
+                             feed_list=[image, label], capacity=4, iterable=True)
 
         self.feature_input = [image]
         self.label_input = label

PaddleCV/PaddleVideo/predict.py

@@ -30,6 +30,7 @@ import models
 from reader import get_reader
 from metrics import get_metrics
 from utils.utility import check_cuda
+from utils.utility import check_version
 
 logging.root.handlers = []
 FORMAT = '[%(levelname)s: %(filename)s: %(lineno)4d]: %(message)s'
@@ -100,7 +101,7 @@ def infer(args):
     infer_config = merge_configs(config, 'infer', vars(args))
     print_configs(infer_config, "Infer")
     infer_model = models.get_model(args.model_name, infer_config, mode='infer')
-    infer_model.build_input(use_pyreader=False)
+    infer_model.build_input(use_dataloader=False)
     infer_model.build_model()
     infer_feeds = infer_model.feeds()
     infer_outputs = infer_model.outputs()
@@ -165,6 +166,7 @@ if __name__ == "__main__":
     args = parse_args()
     # check whether the installed paddle is compiled with GPU
     check_cuda(args.use_gpu)
+    check_version()
     logger.info(args)
 
     infer(args)

PaddleCV/PaddleVideo/train.py

@@ -21,12 +21,13 @@ import logging
 import numpy as np
 import paddle.fluid as fluid
 
-from utils.train_utils import train_with_pyreader
+from utils.train_utils import train_with_dataloader
 import models
 from utils.config_utils import *
 from reader import get_reader
 from metrics import get_metrics
 from utils.utility import check_cuda
+from utils.utility import check_version
 
 logging.root.handlers = []
 FORMAT = '[%(levelname)s: %(filename)s: %(lineno)4d]: %(message)s'
@@ -124,7 +125,7 @@ def train(args):
         train_prog.random_seed = 1000
     with fluid.program_guard(train_prog, startup):
         with fluid.unique_name.guard():
-            train_model.build_input(use_pyreader=True)
+            train_model.build_input(use_dataloader=True)
             train_model.build_model()
             # for the input, has the form [data1, data2,..., label], so train_feeds[-1] is label
             train_feeds = train_model.feeds()
@@ -134,16 +135,16 @@ def train(args):
                 item.persistable = True
             optimizer = train_model.optimizer()
             optimizer.minimize(train_loss)
-            train_pyreader = train_model.pyreader()
+            train_dataloader = train_model.dataloader()
 
     valid_prog = fluid.Program()
     with fluid.program_guard(valid_prog, startup):
         with fluid.unique_name.guard():
-            valid_model.build_input(use_pyreader=True)
+            valid_model.build_input(use_dataloader=True)
             valid_model.build_model()
             valid_feeds = valid_model.feeds()
             valid_fetch_list = valid_model.fetches()
-            valid_pyreader = valid_model.pyreader()
+            valid_dataloader = valid_model.dataloader()
             for item in valid_fetch_list:
                 item.persistable = True
 
@@ -190,8 +191,8 @@ def train(args):
             gpus = gpus.split(",")
             num_gpus = len(gpus)
             assert num_gpus == train_config.TRAIN.num_gpus, \
-                   "num_gpus({}) set by CUDA_VISIBLE_DEVICES" \
-                   "shoud be the same as that" \
+                   "num_gpus({}) set by CUDA_VISIBLE_DEVICES " \
+                   "shoud be the same as that " \
                    "set in {}({})".format(
                    num_gpus, args.config, train_config.TRAIN.num_gpus)
         bs_denominator = train_config.TRAIN.num_gpus
@@ -210,16 +211,16 @@ def train(args):
     epochs = args.epoch or train_model.epoch_num()
 
     exe_places = fluid.cuda_places() if args.use_gpu else fluid.cpu_places()
-    train_pyreader.decorate_sample_list_generator(
+    train_dataloader.set_sample_list_generator(
         train_reader, places=exe_places)
-    valid_pyreader.decorate_sample_list_generator(
+    valid_dataloader.set_sample_list_generator(
         valid_reader, places=exe_places)
 
-    train_with_pyreader(
+    train_with_dataloader(
         exe,
         train_prog,
         compiled_train_prog,  #train_exe,
-        train_pyreader,
+        train_dataloader,
         train_fetch_list,
         train_metrics,
         epochs=epochs,
@@ -229,7 +230,7 @@ def train(args):
         save_model_name=args.model_name,
         fix_random_seed=args.fix_random_seed,
         compiled_test_prog=compiled_valid_prog,  #test_exe=valid_exe,
-        test_pyreader=valid_pyreader,
+        test_dataloader=valid_dataloader,
         test_fetch_list=valid_fetch_list,
         test_metrics=valid_metrics)
 
@@ -238,6 +239,7 @@ if __name__ == "__main__":
     args = parse_args()
     # check whether the installed paddle is compiled with GPU
     check_cuda(args.use_gpu)
+    check_version()
     logger.info(args)
 
     if not os.path.exists(args.save_dir):

PaddleCV/PaddleVideo/utils/train_utils.py

@@ -47,19 +47,19 @@ def log_lr_and_step():
         logger.warn("Unable to get learning_rate and LR_DECAY_COUNTER.")
 
 
-def test_with_pyreader(exe,
+def test_with_dataloader(exe,
                        compiled_test_prog,
-                       test_pyreader,
+                       test_dataloader,
                        test_fetch_list,
                        test_metrics,
                        log_interval=0,
                        save_model_name=''):
-    if not test_pyreader:
-        logger.error("[TEST] get pyreader failed.")
+    if not test_dataloader:
+        logger.error("[TEST] get dataloader failed.")
     test_metrics.reset()
     test_iter = 0
 
-    for data in test_pyreader():
+    for data in test_dataloader():
         test_outs = exe.run(compiled_test_prog,
                             fetch_list=test_fetch_list,
                             feed=data)
@@ -71,14 +71,14 @@ def test_with_pyreader(exe,
     test_metrics.finalize_and_log_out("[TEST] Finish")
 
 
-def train_with_pyreader(exe, train_prog, compiled_train_prog, train_pyreader, \
+def train_with_dataloader(exe, train_prog, compiled_train_prog, train_dataloader, \
                         train_fetch_list, train_metrics, epochs = 10, \
                         log_interval = 0, valid_interval = 0, save_dir = './', \
                         save_model_name = 'model', fix_random_seed = False, \
-                        compiled_test_prog = None, test_pyreader = None, \
+                        compiled_test_prog = None, test_dataloader = None, \
                         test_fetch_list = None, test_metrics = None):
-    if not train_pyreader:
-        logger.error("[TRAIN] get pyreader failed.")
+    if not train_dataloader:
+        logger.error("[TRAIN] get dataloader failed.")
     epoch_periods = []
     train_loss = 0
     for epoch in range(epochs):
@@ -87,7 +87,7 @@ def train_with_pyreader(exe, train_prog, compiled_train_prog, train_pyreader, \
         train_iter = 0
         epoch_periods = []
 
-        for data in train_pyreader():
+        for data in train_dataloader():
             cur_time = time.time()
             train_outs = exe.run(compiled_train_prog,
                                  fetch_list=train_fetch_list,
@@ -122,7 +122,7 @@ def train_with_pyreader(exe, train_prog, compiled_train_prog, train_pyreader, \
             save_type='.pdparams')
         if compiled_test_prog and valid_interval > 0 and (
                 epoch + 1) % valid_interval == 0:
-            test_with_pyreader(exe, compiled_test_prog, test_pyreader,
+            test_with_dataloader(exe, compiled_test_prog, test_dataloader,
                                test_fetch_list, test_metrics, log_interval,
                                save_model_name)
 

PaddleCV/PaddleVideo/utils/utility.py

@@ -14,11 +14,13 @@
 
 import os
 import signal
+import logging
 import paddle
 import paddle.fluid as fluid
 
 __all__ = ['AttrDict']
 
+logger = logging.getLogger(__name__)
 
 def _term(sig_num, addition):
     print('current pid is %s, group id is %s' % (os.getpid(), os.getpgrp()))
@@ -49,3 +51,18 @@ def check_cuda(use_cuda, err = \
             sys.exit(1)
     except Exception as e:
         pass
+
+def check_version():
+     """
+     Log error and exit when the installed version of paddlepaddle is
+     not satisfied.
+     """
+     err = "PaddlePaddle version 1.6 or higher is required, " \
+           "or a suitable develop version is satisfied as well. \n" \
+           "Please make sure the version is good with your code." \
+
+     try:
+         fluid.require_version('1.6.0')
+     except Exception as e:
+         logger.error(err)
+         sys.exit(1)