From 1137c0c0a2f33592d3a3191b0c74cb8e897e3bdb Mon Sep 17 00:00:00 2001 From: SunGaofeng Date: Mon, 14 Oct 2019 03:05:39 -0500 Subject: [PATCH] Revert "fix fluid.data and fluid.io.DataLoader (#3533)" (#3560) This reverts commit 0be3697cb4ef9f7bdf0596473236bf49055cf9cf. --- PaddleCV/PaddleVideo/README.md | 2 +- PaddleCV/PaddleVideo/eval.py | 4 +- PaddleCV/PaddleVideo/inference_model.py | 2 +- .../attention_cluster/attention_cluster.py | 16 ++--- .../models/attention_lstm/attention_lstm.py | 16 ++--- PaddleCV/PaddleVideo/models/bmn/bmn.py | 39 +++++----- PaddleCV/PaddleVideo/models/bsn/bsn.py | 72 ++++++++++--------- PaddleCV/PaddleVideo/models/ctcn/ctcn.py | 33 ++++----- PaddleCV/PaddleVideo/models/model.py | 8 +-- .../PaddleVideo/models/nextvlad/nextvlad.py | 22 +++--- .../models/nonlocal_model/nonlocal_model.py | 18 ++--- PaddleCV/PaddleVideo/models/stnet/stnet.py | 20 +++--- PaddleCV/PaddleVideo/models/tsm/tsm.py | 19 ++--- PaddleCV/PaddleVideo/models/tsn/tsn.py | 19 ++--- PaddleCV/PaddleVideo/predict.py | 4 +- PaddleCV/PaddleVideo/train.py | 26 ++++--- PaddleCV/PaddleVideo/utils/train_utils.py | 22 +++--- PaddleCV/PaddleVideo/utils/utility.py | 17 ----- 18 files changed, 171 insertions(+), 188 deletions(-) diff --git a/PaddleCV/PaddleVideo/README.md b/PaddleCV/PaddleVideo/README.md index e277fb4a..e0d55ba7 100644 --- a/PaddleCV/PaddleVideo/README.md +++ b/PaddleCV/PaddleVideo/README.md @@ -25,7 +25,7 @@ ## 安装 -在当前模型库运行样例代码需要PaddlePaddle Fluid v.1.6.0或以上的版本。如果你的运行环境中的PaddlePaddle低于此版本，请根据[安装文档](http://www.paddlepaddle.org/documentation/docs/zh/1.6/beginners_guide/install/index_cn.html)中的说明来更新PaddlePaddle。 +在当前模型库运行样例代码需要PaddlePaddle Fluid v.1.5.0或以上的版本。如果你的运行环境中的PaddlePaddle低于此版本，请根据[安装文档](http://www.paddlepaddle.org/documentation/docs/zh/1.5/beginners_guide/install/index_cn.html)中的说明来更新PaddlePaddle。 ### 其他环境依赖 diff --git a/PaddleCV/PaddleVideo/eval.py b/PaddleCV/PaddleVideo/eval.py index 14ab4bb7..3c2641d0 100644 --- a/PaddleCV/PaddleVideo/eval.py +++ b/PaddleCV/PaddleVideo/eval.py @@ -26,7 +26,6 @@ import models from reader import get_reader from metrics import get_metrics from utils.utility import check_cuda -from utils.utility import check_version logging.root.handlers = [] FORMAT = '[%(levelname)s: %(filename)s: %(lineno)4d]: %(message)s' @@ -84,7 +83,7 @@ def test(args): # build model test_model = models.get_model(args.model_name, test_config, mode='test') - test_model.build_input(use_dataloader=False) + test_model.build_input(use_pyreader=False) test_model.build_model() test_feeds = test_model.feeds() test_fetch_list = test_model.fetches() @@ -131,7 +130,6 @@ if __name__ == "__main__": args = parse_args() # check whether the installed paddle is compiled with GPU check_cuda(args.use_gpu) - check_version() logger.info(args) test(args) diff --git a/PaddleCV/PaddleVideo/inference_model.py b/PaddleCV/PaddleVideo/inference_model.py index 9ed5d272..5fe8ec14 100644 --- a/PaddleCV/PaddleVideo/inference_model.py +++ b/PaddleCV/PaddleVideo/inference_model.py @@ -80,7 +80,7 @@ def save_inference_model(args): infer_config = merge_configs(config, 'infer', vars(args)) print_configs(infer_config, "Infer") infer_model = models.get_model(args.model_name, infer_config, mode='infer') - infer_model.build_input(use_dataloader=False) + infer_model.build_input(use_pyreader=False) infer_model.build_model() infer_feeds = infer_model.feeds() infer_outputs = infer_model.outputs() diff --git a/PaddleCV/PaddleVideo/models/attention_cluster/attention_cluster.py b/PaddleCV/PaddleVideo/models/attention_cluster/attention_cluster.py index fbc8a0c1..555008af 100644 --- a/PaddleCV/PaddleVideo/models/attention_cluster/attention_cluster.py +++ b/PaddleCV/PaddleVideo/models/attention_cluster/attention_cluster.py @@ -41,21 +41,21 @@ class AttentionCluster(ModelBase): self.learning_rate = self.get_config_from_sec('train', 'learning_rate', 1e-3) - def build_input(self, use_dataloader=True): + def build_input(self, use_pyreader=True): self.feature_input = [] for name, dim in zip(self.feature_names, self.feature_dims): self.feature_input.append( - fluid.data( - shape=[None, self.seg_num, dim], dtype='float32', name=name)) + fluid.layers.data( + shape=[self.seg_num, dim], dtype='float32', name=name)) if self.mode != 'infer': - self.label_input = fluid.data( - shape=[None, self.class_num], dtype='float32', name='label') + self.label_input = fluid.layers.data( + shape=[self.class_num], dtype='float32', name='label') else: self.label_input = None - if use_dataloader: + if use_pyreader: assert self.mode != 'infer', \ - 'dataloader is not recommendated when infer, please set use_dataloader to be false.' - self.dataloader = fluid.io.DataLoader.from_generator( + 'pyreader is not recommendated when infer, please set use_pyreader to be false.' + self.py_reader = fluid.io.PyReader( feed_list=self.feature_input + [self.label_input], capacity=8, iterable=True) diff --git a/PaddleCV/PaddleVideo/models/attention_lstm/attention_lstm.py b/PaddleCV/PaddleVideo/models/attention_lstm/attention_lstm.py index d9acd194..ede38b8e 100644 --- a/PaddleCV/PaddleVideo/models/attention_lstm/attention_lstm.py +++ b/PaddleCV/PaddleVideo/models/attention_lstm/attention_lstm.py @@ -52,21 +52,21 @@ class AttentionLSTM(ModelBase): self.decay_gamma = self.get_config_from_sec('train', 'decay_gamma', 0.1) - def build_input(self, use_dataloader): + def build_input(self, use_pyreader): self.feature_input = [] for name, dim in zip(self.feature_names, self.feature_dims): self.feature_input.append( - fluid.data( - shape=[None, dim], lod_level=1, dtype='float32', name=name)) + fluid.layers.data( + shape=[dim], lod_level=1, dtype='float32', name=name)) if self.mode != 'infer': - self.label_input = fluid.data( - shape=[None, self.num_classes], dtype='float32', name='label') + self.label_input = fluid.layers.data( + shape=[self.num_classes], dtype='float32', name='label') else: self.label_input = None - if use_dataloader: + if use_pyreader: assert self.mode != 'infer', \ - 'dataloader is not recommendated when infer, please set use_dataloader to be false.' - self.dataloader = fluid.io.DataLoader.from_generator( + 'pyreader is not recommendated when infer, please set use_pyreader to be false.' + self.py_reader = fluid.io.PyReader( feed_list=self.feature_input + [self.label_input], capacity=8, iterable=True) diff --git a/PaddleCV/PaddleVideo/models/bmn/bmn.py b/PaddleCV/PaddleVideo/models/bmn/bmn.py index 6c5bccaa..aae23da3 100644 --- a/PaddleCV/PaddleVideo/models/bmn/bmn.py +++ b/PaddleCV/PaddleVideo/models/bmn/bmn.py @@ -52,46 +52,47 @@ class BMN(ModelBase): 'l2_weight_decay') self.lr_decay_iter = self.get_config_from_sec('train', 'lr_decay_iter') - def build_input(self, use_dataloader=True): - feat_shape = [None, self.feat_dim, self.tscale] - gt_iou_map_shape = [None, self.dscale, self.tscale] - gt_start_shape = [None, self.tscale] - gt_end_shape = [None, self.tscale] - fileid_shape = [None, 1] - self.use_dataloader = use_dataloader + def build_input(self, use_pyreader=True): + feat_shape = [self.feat_dim, self.tscale] + gt_iou_map_shape = [self.dscale, self.tscale] + gt_start_shape = [self.tscale] + gt_end_shape = [self.tscale] + fileid_shape = [1] + self.use_pyreader = use_pyreader # set init data to None + py_reader = None feat = None gt_iou_map = None gt_start = None gt_end = None fileid = None - feat = fluid.data(name='feat', shape=feat_shape, dtype='float32') + feat = fluid.layers.data(name='feat', shape=feat_shape, dtype='float32') feed_list = [] feed_list.append(feat) if (self.mode == 'train') or (self.mode == 'valid'): - gt_start = fluid.data( + gt_start = fluid.layers.data( name='gt_start', shape=gt_start_shape, dtype='float32') - gt_end = fluid.data( + gt_end = fluid.layers.data( name='gt_end', shape=gt_end_shape, dtype='float32') - gt_iou_map = fluid.data( + gt_iou_map = fluid.layers.data( name='gt_iou_map', shape=gt_iou_map_shape, dtype='float32') feed_list.append(gt_iou_map) feed_list.append(gt_start) feed_list.append(gt_end) elif self.mode == 'test': - gt_start = fluid.data( + gt_start = fluid.layers.data( name='gt_start', shape=gt_start_shape, dtype='float32') - gt_end = fluid.data( + gt_end = fluid.layers.data( name='gt_end', shape=gt_end_shape, dtype='float32') - gt_iou_map = fluid.data( + gt_iou_map = fluid.layers.data( name='gt_iou_map', shape=gt_iou_map_shape, dtype='float32') feed_list.append(gt_iou_map) feed_list.append(gt_start) feed_list.append(gt_end) - fileid = fluid.data( + fileid = fluid.layers.data( name='fileid', shape=fileid_shape, dtype='int64') feed_list.append(fileid) elif self.mode == 'infer': @@ -101,11 +102,11 @@ class BMN(ModelBase): raise NotImplementedError('mode {} not implemented'.format( self.mode)) - if use_dataloader: + if use_pyreader: assert self.mode != 'infer', \ - 'dataloader is not recommendated when infer, please set use_dataloader to be false.' - self.dataloader = fluid.io.DataLoader.from_generator( - feed_list=feed_list, capacity=8, iterable=True) + 'pyreader is not recommendated when infer, please set use_pyreader to be false.' + self.py_reader = fluid.io.PyReader( + feed_list=feed_list, capacity=8, iterable=True) self.feat_input = [feat] self.gt_iou_map = gt_iou_map diff --git a/PaddleCV/PaddleVideo/models/bsn/bsn.py b/PaddleCV/PaddleVideo/models/bsn/bsn.py index cf23dc1f..1fc66504 100644 --- a/PaddleCV/PaddleVideo/models/bsn/bsn.py +++ b/PaddleCV/PaddleVideo/models/bsn/bsn.py @@ -47,46 +47,47 @@ class BsnTem(ModelBase): 'l2_weight_decay') self.lr_decay_iter = self.get_config_from_sec('train', 'lr_decay_iter') - def build_input(self, use_dataloader=True): - feat_shape = [None, self.feat_dim, self.tscale] - gt_start_shape = [None, self.tscale] - gt_end_shape = [None, self.tscale] - gt_action_shape = [None, self.tscale] - fileid_shape = [None, 1] - self.use_dataloader = use_dataloader + def build_input(self, use_pyreader=True): + feat_shape = [self.feat_dim, self.tscale] + gt_start_shape = [self.tscale] + gt_end_shape = [self.tscale] + gt_action_shape = [self.tscale] + fileid_shape = [1] + self.use_pyreader = use_pyreader # set init data to None + py_reader = None feat = None gt_start = None gt_end = None gt_action = None fileid = None - feat = fluid.data(name='feat', shape=feat_shape, dtype='float32') + feat = fluid.layers.data(name='feat', shape=feat_shape, dtype='float32') feed_list = [] feed_list.append(feat) if (self.mode == 'train') or (self.mode == 'valid'): - gt_start = fluid.data( + gt_start = fluid.layers.data( name='gt_start', shape=gt_start_shape, dtype='float32') - gt_end = fluid.data( + gt_end = fluid.layers.data( name='gt_end', shape=gt_end_shape, dtype='float32') - gt_action = fluid.data( + gt_action = fluid.layers.data( name='gt_action', shape=gt_action_shape, dtype='float32') feed_list.append(gt_start) feed_list.append(gt_end) feed_list.append(gt_action) elif self.mode == 'test': - gt_start = fluid.data( + gt_start = fluid.layers.data( name='gt_start', shape=gt_start_shape, dtype='float32') - gt_end = fluid.data( + gt_end = fluid.layers.data( name='gt_end', shape=gt_end_shape, dtype='float32') - gt_action = fluid.data( + gt_action = fluid.layers.data( name='gt_action', shape=gt_action_shape, dtype='float32') feed_list.append(gt_start) feed_list.append(gt_end) feed_list.append(gt_action) - fileid = fluid.data( + fileid = fluid.layers.data( name='fileid', shape=fileid_shape, dtype='int64') feed_list.append(fileid) elif self.mode == 'infer': @@ -96,11 +97,11 @@ class BsnTem(ModelBase): raise NotImplementedError('mode {} not implemented'.format( self.mode)) - if use_dataloader: + if use_pyreader: assert self.mode != 'infer', \ - 'dataloader is not recommendated when infer, please set use_dataloader to be false.' - self.dataloader = fluid.io.DataLoader.from_generator( - feed_list=feed_list, capacity=8, iterable=True) + 'pyreader is not recommendated when infer, please set use_pyreader to be false.' + self.py_reader = fluid.io.PyReader( + feed_list=feed_list, capacity=8, iterable=True) self.feat_input = [feat] self.gt_start = gt_start @@ -211,50 +212,51 @@ class BsnPem(ModelBase): 'l2_weight_decay') self.lr_decay_iter = self.get_config_from_sec('train', 'lr_decay_iter') - def build_input(self, use_dataloader=True): - feat_shape = [None, self.top_K, self.feat_dim] - gt_iou_shape = [None. self.top_K, 1] - props_info_shape = [None, self.top_K, 4] - fileid_shape = [None, 1] - self.use_dataloader = use_dataloader + def build_input(self, use_pyreader=True): + feat_shape = [self.top_K, self.feat_dim] + gt_iou_shape = [self.top_K, 1] + props_info_shape = [self.top_K, 4] + fileid_shape = [1] + self.use_pyreader = use_pyreader # set init data to None + py_reader = None feat = None gt_iou = None props_info = None fileid = None - feat = fluid.data(name='feat', shape=feat_shape, dtype='float32') + feat = fluid.layers.data(name='feat', shape=feat_shape, dtype='float32') feed_list = [] feed_list.append(feat) if (self.mode == 'train') or (self.mode == 'valid'): - gt_iou = fluid.data( + gt_iou = fluid.layers.data( name='gt_iou', shape=gt_iou_shape, dtype='float32') feed_list.append(gt_iou) elif self.mode == 'test': - gt_iou = fluid.data( + gt_iou = fluid.layers.data( name='gt_iou', shape=gt_iou_shape, dtype='float32') - props_info = fluid.data( + props_info = fluid.layers.data( name='props_info', shape=props_info_shape, dtype='float32') feed_list.append(gt_iou) feed_list.append(props_info) - fileid = fluid.data( + fileid = fluid.layers.data( name='fileid', shape=fileid_shape, dtype='int64') feed_list.append(fileid) elif self.mode == 'infer': - props_info = fluid.data( + props_info = fluid.layers.data( name='props_info', shape=props_info_shape, dtype='float32') feed_list.append(props_info) else: raise NotImplementedError('mode {} not implemented'.format( self.mode)) - if use_dataloader: + if use_pyreader: assert self.mode != 'infer', \ - 'dataloader is not recommendated when infer, please set use_dataloader to be false.' - self.dataloader = fluid.io.DataLoader.from_generator( - feed_list=feed_list, capacity=4, iterable=True) + 'pyreader is not recommendated when infer, please set use_pyreader to be false.' + self.py_reader = fluid.io.PyReader( + feed_list=feed_list, capacity=4, iterable=True) self.feat_input = [feat] self.gt_iou = gt_iou diff --git a/PaddleCV/PaddleVideo/models/ctcn/ctcn.py b/PaddleCV/PaddleVideo/models/ctcn/ctcn.py index 1fdd06ca..3fa6e55a 100644 --- a/PaddleCV/PaddleVideo/models/ctcn/ctcn.py +++ b/PaddleCV/PaddleVideo/models/ctcn/ctcn.py @@ -51,36 +51,37 @@ class CTCN(ModelBase): self.momentum = self.get_config_from_sec('train', 'momentum') self.lr_decay_iter = self.get_config_from_sec('train', 'lr_decay_iter') - def build_input(self, use_dataloader=True): - image_shape = [None, 1, self.img_size, self.concept_size] - loc_shape = [None, self.total_num_anchors, 2] - cls_shape = [None, self.total_num_anchors] - fileid_shape = [None, 1] - self.use_dataloader = use_dataloader + def build_input(self, use_pyreader=True): + image_shape = [1, self.img_size, self.concept_size] + loc_shape = [self.total_num_anchors, 2] + cls_shape = [self.total_num_anchors] + fileid_shape = [1] + self.use_pyreader = use_pyreader # set init data to None + py_reader = None image = None loc_targets = None cls_targets = None fileid = None - image = fluid.data( + image = fluid.layers.data( name='image', shape=image_shape, dtype='float32') feed_list = [] feed_list.append(image) if (self.mode == 'train') or (self.mode == 'valid'): - loc_targets = fluid.data( + loc_targets = fluid.layers.data( name='loc_targets', shape=loc_shape, dtype='float32') - cls_targets = fluid.data( + cls_targets = fluid.layers.data( name='cls_targets', shape=cls_shape, dtype='int64') feed_list.append(loc_targets) feed_list.append(cls_targets) elif self.mode == 'test': - loc_targets = fluid.data( + loc_targets = fluid.layers.data( name='loc_targets', shape=loc_shape, dtype='float32') - cls_targets = fluid.data( + cls_targets = fluid.layers.data( name='cls_targets', shape=cls_shape, dtype='int64') - fileid = fluid.data( + fileid = fluid.layers.data( name='fileid', shape=fileid_shape, dtype='int64') feed_list.append(loc_targets) feed_list.append(cls_targets) @@ -92,11 +93,11 @@ class CTCN(ModelBase): raise NotImplementedError('mode {} not implemented'.format( self.mode)) - if use_dataloader: + if use_pyreader: assert self.mode != 'infer', \ - 'dataloader is not recommendated when infer, please set use_dataloader to be false.' - self.dataloader = fluid.io.DataLoader.from_generator( - feed_list=feed_list, capacity=4, iterable=True) + 'pyreader is not recommendated when infer, please set use_pyreader to be false.' + self.py_reader = fluid.io.PyReader( + feed_list=feed_list, capacity=4, iterable=True) self.feature_input = [image] self.cls_targets = cls_targets diff --git a/PaddleCV/PaddleVideo/models/model.py b/PaddleCV/PaddleVideo/models/model.py index 29b132fe..fd04dfde 100644 --- a/PaddleCV/PaddleVideo/models/model.py +++ b/PaddleCV/PaddleVideo/models/model.py @@ -65,13 +65,13 @@ class ModelBase(object): self.is_training = (mode == 'train') self.mode = mode self.cfg = cfg - self.dataloader = None + self.py_reader = None def build_model(self): "build model struct" raise NotImplementError(self, self.build_model) - def build_input(self, use_dataloader): + def build_input(self, use_pyreader): "build input Variable" raise NotImplementError(self, self.build_input) @@ -114,8 +114,8 @@ class ModelBase(object): wget.download(url, path) return path - def dataloader(self): - return self.dataloader + def pyreader(self): + return self.py_reader def epoch_num(self): "get train epoch num" diff --git a/PaddleCV/PaddleVideo/models/nextvlad/nextvlad.py b/PaddleCV/PaddleVideo/models/nextvlad/nextvlad.py index 4b06a560..295165fc 100644 --- a/PaddleCV/PaddleVideo/models/nextvlad/nextvlad.py +++ b/PaddleCV/PaddleVideo/models/nextvlad/nextvlad.py @@ -61,17 +61,17 @@ class NEXTVLAD(ModelBase): # other params self.batch_size = self.get_config_from_sec(self.mode, 'batch_size') - def build_input(self, use_dataloader=True): - rgb_shape = [None, self.video_feature_size] - audio_shape = [None, self.audio_feature_size] - label_shape = [None, self.num_classes] + def build_input(self, use_pyreader=True): + rgb_shape = [self.video_feature_size] + audio_shape = [self.audio_feature_size] + label_shape = [self.num_classes] - rgb = fluid.data( + rgb = fluid.layers.data( name='train_rgb' if self.is_training else 'test_rgb', shape=rgb_shape, dtype='uint8', lod_level=1) - audio = fluid.data( + audio = fluid.layers.data( name='train_audio' if self.is_training else 'test_audio', shape=audio_shape, dtype='uint8', @@ -79,16 +79,16 @@ class NEXTVLAD(ModelBase): if self.mode == 'infer': label = None else: - label = fluid.data( + label = fluid.layers.data( name='train_label' if self.is_training else 'test_label', shape=label_shape, dtype='float32') - if use_dataloader: + if use_pyreader: assert self.mode != 'infer', \ - 'dataloader is not recommendated when infer, please set use_dataloader to be false.' - self.dataloader = fluid.io.DataLoader.from_generator( - feed_list=[rgb, audio, label], capacity=8, iterable=True) + 'pyreader is not recommendated when infer, please set use_pyreader to be false.' + self.py_reader = fluid.io.PyReader( + feed_list=[rgb, audio, label], capacity=8, iterable=True) self.feature_input = [rgb, audio] self.label_input = label diff --git a/PaddleCV/PaddleVideo/models/nonlocal_model/nonlocal_model.py b/PaddleCV/PaddleVideo/models/nonlocal_model/nonlocal_model.py index aa75c35d..5555ee46 100644 --- a/PaddleCV/PaddleVideo/models/nonlocal_model/nonlocal_model.py +++ b/PaddleCV/PaddleVideo/models/nonlocal_model/nonlocal_model.py @@ -39,27 +39,27 @@ class NonLocal(ModelBase): # crop size self.crop_size = self.get_config_from_sec(self.mode, 'crop_size') - def build_input(self, use_dataloader=True): - input_shape = [None, 3, self.video_length, self.crop_size, self.crop_size] - label_shape = [None, 1] + def build_input(self, use_pyreader=True): + input_shape = [3, self.video_length, self.crop_size, self.crop_size] + label_shape = [1] - data = fluid.data( + data = fluid.layers.data( name='train_data' if self.is_training else 'test_data', shape=input_shape, dtype='float32') if self.mode != 'infer': - label = fluid.data( + label = fluid.layers.data( name='train_label' if self.is_training else 'test_label', shape=label_shape, dtype='int64') else: label = None - if use_dataloader: + if use_pyreader: assert self.mode != 'infer', \ - 'dataloader is not recommendated when infer, please set use_dataloader to be false.' - self.dataloader = fluid.io.DataLoader.from_generator( - feed_list=[data, label], capacity=4, iterable=True) + 'pyreader is not recommendated when infer, please set use_pyreader to be false.' + self.py_reader = fluid.io.PyReader( + feed_list=[data, label], capacity=4, iterable=True) self.feature_input = [data] self.label_input = label diff --git a/PaddleCV/PaddleVideo/models/stnet/stnet.py b/PaddleCV/PaddleVideo/models/stnet/stnet.py index 343862a6..c257df1f 100644 --- a/PaddleCV/PaddleVideo/models/stnet/stnet.py +++ b/PaddleCV/PaddleVideo/models/stnet/stnet.py @@ -51,24 +51,25 @@ class STNET(ModelBase): self.target_size = self.get_config_from_sec(self.mode, 'target_size') self.batch_size = self.get_config_from_sec(self.mode, 'batch_size') - def build_input(self, use_dataloader=True): + def build_input(self, use_pyreader=True): image_shape = [3, self.target_size, self.target_size] image_shape[0] = image_shape[0] * self.seglen - image_shape = [None, self.seg_num] + image_shape - self.use_dataloader = use_dataloader + image_shape = [self.seg_num] + image_shape + self.use_pyreader = use_pyreader - image = fluid.data( + image = fluid.layers.data( name='image', shape=image_shape, dtype='float32') if self.mode != 'infer': - label = fluid.data(name='label', shape=[None, 1], dtype='int64') + label = fluid.layers.data(name='label', shape=[1], dtype='int64') else: label = None - if use_dataloader: + if use_pyreader: assert self.mode != 'infer', \ - 'dataloader is not recommendated when infer, please set use_dataloader to be false.' - self.dataloader = fluid.io.DataLoader.from_generator( - feed_list=[image, label], capacity=4, iterable=True) + 'pyreader is not recommendated when infer, please set use_pyreader to be false.' + py_reader = fluid.io.PyReader( + feed_list=[image, label], capacity=4, iterable=True) + self.py_reader = py_reader self.feature_input = [image] self.label_input = label @@ -166,4 +167,3 @@ class STNET(ModelBase): param_numpy = np.mean(param_numpy, axis=1, keepdims=True) / self.seglen param_numpy = np.repeat(param_numpy, 3 * self.seglen, axis=1) param_tensor.set(param_numpy.astype(np.float32), place) - diff --git a/PaddleCV/PaddleVideo/models/tsm/tsm.py b/PaddleCV/PaddleVideo/models/tsm/tsm.py index 07f3fd21..bd69d0ef 100644 --- a/PaddleCV/PaddleVideo/models/tsm/tsm.py +++ b/PaddleCV/PaddleVideo/models/tsm/tsm.py @@ -51,24 +51,25 @@ class TSM(ModelBase): self.target_size = self.get_config_from_sec(self.mode, 'target_size') self.batch_size = self.get_config_from_sec(self.mode, 'batch_size') - def build_input(self, use_dataloader=True): + def build_input(self, use_pyreader=True): image_shape = [3, self.target_size, self.target_size] image_shape[0] = image_shape[0] * self.seglen - image_shape = [None, self.seg_num] + image_shape - self.use_dataloader = use_dataloader + image_shape = [self.seg_num] + image_shape + self.use_pyreader = use_pyreader - image = fluid.data( + image = fluid.layers.data( name='image', shape=image_shape, dtype='float32') if self.mode != 'infer': - label = fluid.data(name='label', shape=[None, 1], dtype='int64') + label = fluid.layers.data(name='label', shape=[1], dtype='int64') else: label = None - if use_dataloader: + if use_pyreader: assert self.mode != 'infer', \ - 'dataloader is not recommendated when infer, please set use_dataloader to be false.' - self.dataloader = fluid.io.DataLoader.from_generator( - feed_list=[image, label], capacity=4, iterable=True) + 'pyreader is not recommendated when infer, please set use_pyreader to be false.' + py_reader = fluid.io.PyReader( + feed_list=[image, label], capacity=4, iterable=True) + self.py_reader = py_reader self.feature_input = [image] self.label_input = label diff --git a/PaddleCV/PaddleVideo/models/tsn/tsn.py b/PaddleCV/PaddleVideo/models/tsn/tsn.py index 337949db..4d3cdc89 100644 --- a/PaddleCV/PaddleVideo/models/tsn/tsn.py +++ b/PaddleCV/PaddleVideo/models/tsn/tsn.py @@ -52,24 +52,25 @@ class TSN(ModelBase): self.target_size = self.get_config_from_sec(self.mode, 'target_size') self.batch_size = self.get_config_from_sec(self.mode, 'batch_size') - def build_input(self, use_dataloader=True): + def build_input(self, use_pyreader=True): image_shape = [3, self.target_size, self.target_size] image_shape[0] = image_shape[0] * self.seglen - image_shape = [None, self.seg_num] + image_shape - self.use_dataloader = use_dataloader + image_shape = [self.seg_num] + image_shape + self.use_pyreader = use_pyreader - image = fluid.data( + image = fluid.layers.data( name='image', shape=image_shape, dtype='float32') if self.mode != 'infer': - label = fluid.data(name='label', shape=[None, 1], dtype='int64') + label = fluid.layers.data(name='label', shape=[1], dtype='int64') else: label = None - if use_dataloader: + if use_pyreader: assert self.mode != 'infer', \ - 'dataloader is not recommendated when infer, please set use_dataloader to be false.' - self.dataloader = fluid.io.DataLoader.from_generator( - feed_list=[image, label], capacity=4, iterable=True) + 'pyreader is not recommendated when infer, please set use_pyreader to be false.' + py_reader = fluid.io.PyReader( + feed_list=[image, label], capacity=4, iterable=True) + self.py_reader = py_reader self.feature_input = [image] self.label_input = label diff --git a/PaddleCV/PaddleVideo/predict.py b/PaddleCV/PaddleVideo/predict.py index 1e5067eb..8eb58fab 100644 --- a/PaddleCV/PaddleVideo/predict.py +++ b/PaddleCV/PaddleVideo/predict.py @@ -30,7 +30,6 @@ import models from reader import get_reader from metrics import get_metrics from utils.utility import check_cuda -from utils.utility import check_version logging.root.handlers = [] FORMAT = '[%(levelname)s: %(filename)s: %(lineno)4d]: %(message)s' @@ -101,7 +100,7 @@ def infer(args): infer_config = merge_configs(config, 'infer', vars(args)) print_configs(infer_config, "Infer") infer_model = models.get_model(args.model_name, infer_config, mode='infer') - infer_model.build_input(use_dataloader=False) + infer_model.build_input(use_pyreader=False) infer_model.build_model() infer_feeds = infer_model.feeds() infer_outputs = infer_model.outputs() @@ -166,7 +165,6 @@ if __name__ == "__main__": args = parse_args() # check whether the installed paddle is compiled with GPU check_cuda(args.use_gpu) - check_version() logger.info(args) infer(args) diff --git a/PaddleCV/PaddleVideo/train.py b/PaddleCV/PaddleVideo/train.py index f06e1885..6ef853b5 100644 --- a/PaddleCV/PaddleVideo/train.py +++ b/PaddleCV/PaddleVideo/train.py @@ -21,13 +21,12 @@ import logging import numpy as np import paddle.fluid as fluid -from utils.train_utils import train_with_dataloader +from utils.train_utils import train_with_pyreader import models from utils.config_utils import * from reader import get_reader from metrics import get_metrics from utils.utility import check_cuda -from utils.utility import check_version logging.root.handlers = [] FORMAT = '[%(levelname)s: %(filename)s: %(lineno)4d]: %(message)s' @@ -125,7 +124,7 @@ def train(args): train_prog.random_seed = 1000 with fluid.program_guard(train_prog, startup): with fluid.unique_name.guard(): - train_model.build_input(use_dataloader=True) + train_model.build_input(use_pyreader=True) train_model.build_model() # for the input, has the form [data1, data2,..., label], so train_feeds[-1] is label train_feeds = train_model.feeds() @@ -135,16 +134,16 @@ def train(args): item.persistable = True optimizer = train_model.optimizer() optimizer.minimize(train_loss) - train_dataloader = train_model.dataloader() + train_pyreader = train_model.pyreader() valid_prog = fluid.Program() with fluid.program_guard(valid_prog, startup): with fluid.unique_name.guard(): - valid_model.build_input(use_dataloader=True) + valid_model.build_input(use_pyreader=True) valid_model.build_model() valid_feeds = valid_model.feeds() valid_fetch_list = valid_model.fetches() - valid_dataloader = valid_model.dataloader() + valid_pyreader = valid_model.pyreader() for item in valid_fetch_list: item.persistable = True @@ -191,8 +190,8 @@ def train(args): gpus = gpus.split(",") num_gpus = len(gpus) assert num_gpus == train_config.TRAIN.num_gpus, \ - "num_gpus({}) set by CUDA_VISIBLE_DEVICES " \ - "shoud be the same as that " \ + "num_gpus({}) set by CUDA_VISIBLE_DEVICES" \ + "shoud be the same as that" \ "set in {}({})".format( num_gpus, args.config, train_config.TRAIN.num_gpus) bs_denominator = train_config.TRAIN.num_gpus @@ -211,16 +210,16 @@ def train(args): epochs = args.epoch or train_model.epoch_num() exe_places = fluid.cuda_places() if args.use_gpu else fluid.cpu_places() - train_dataloader.set_sample_list_generator( + train_pyreader.decorate_sample_list_generator( train_reader, places=exe_places) - valid_dataloader.set_sample_list_generator( + valid_pyreader.decorate_sample_list_generator( valid_reader, places=exe_places) - train_with_dataloader( + train_with_pyreader( exe, train_prog, compiled_train_prog, #train_exe, - train_dataloader, + train_pyreader, train_fetch_list, train_metrics, epochs=epochs, @@ -230,7 +229,7 @@ def train(args): save_model_name=args.model_name, fix_random_seed=args.fix_random_seed, compiled_test_prog=compiled_valid_prog, #test_exe=valid_exe, - test_dataloader=valid_dataloader, + test_pyreader=valid_pyreader, test_fetch_list=valid_fetch_list, test_metrics=valid_metrics) @@ -239,7 +238,6 @@ if __name__ == "__main__": args = parse_args() # check whether the installed paddle is compiled with GPU check_cuda(args.use_gpu) - check_version() logger.info(args) if not os.path.exists(args.save_dir): diff --git a/PaddleCV/PaddleVideo/utils/train_utils.py b/PaddleCV/PaddleVideo/utils/train_utils.py index 817ab9f5..abec586f 100644 --- a/PaddleCV/PaddleVideo/utils/train_utils.py +++ b/PaddleCV/PaddleVideo/utils/train_utils.py @@ -47,19 +47,19 @@ def log_lr_and_step(): logger.warn("Unable to get learning_rate and LR_DECAY_COUNTER.") -def test_with_dataloader(exe, +def test_with_pyreader(exe, compiled_test_prog, - test_dataloader, + test_pyreader, test_fetch_list, test_metrics, log_interval=0, save_model_name=''): - if not test_dataloader: - logger.error("[TEST] get dataloader failed.") + if not test_pyreader: + logger.error("[TEST] get pyreader failed.") test_metrics.reset() test_iter = 0 - for data in test_dataloader(): + for data in test_pyreader(): test_outs = exe.run(compiled_test_prog, fetch_list=test_fetch_list, feed=data) @@ -71,14 +71,14 @@ def test_with_dataloader(exe, test_metrics.finalize_and_log_out("[TEST] Finish") -def train_with_dataloader(exe, train_prog, compiled_train_prog, train_dataloader, \ +def train_with_pyreader(exe, train_prog, compiled_train_prog, train_pyreader, \ train_fetch_list, train_metrics, epochs = 10, \ log_interval = 0, valid_interval = 0, save_dir = './', \ save_model_name = 'model', fix_random_seed = False, \ - compiled_test_prog = None, test_dataloader = None, \ + compiled_test_prog = None, test_pyreader = None, \ test_fetch_list = None, test_metrics = None): - if not train_dataloader: - logger.error("[TRAIN] get dataloader failed.") + if not train_pyreader: + logger.error("[TRAIN] get pyreader failed.") epoch_periods = [] train_loss = 0 for epoch in range(epochs): @@ -87,7 +87,7 @@ def train_with_dataloader(exe, train_prog, compiled_train_prog, train_dataloader train_iter = 0 epoch_periods = [] - for data in train_dataloader(): + for data in train_pyreader(): cur_time = time.time() train_outs = exe.run(compiled_train_prog, fetch_list=train_fetch_list, @@ -122,7 +122,7 @@ def train_with_dataloader(exe, train_prog, compiled_train_prog, train_dataloader save_type='.pdparams') if compiled_test_prog and valid_interval > 0 and ( epoch + 1) % valid_interval == 0: - test_with_dataloader(exe, compiled_test_prog, test_dataloader, + test_with_pyreader(exe, compiled_test_prog, test_pyreader, test_fetch_list, test_metrics, log_interval, save_model_name) diff --git a/PaddleCV/PaddleVideo/utils/utility.py b/PaddleCV/PaddleVideo/utils/utility.py index ea3f580d..02414386 100644 --- a/PaddleCV/PaddleVideo/utils/utility.py +++ b/PaddleCV/PaddleVideo/utils/utility.py @@ -14,13 +14,11 @@ import os import signal -import logging import paddle import paddle.fluid as fluid __all__ = ['AttrDict'] -logger = logging.getLogger(__name__) def _term(sig_num, addition): print('current pid is %s, group id is %s' % (os.getpid(), os.getpgrp())) @@ -51,18 +49,3 @@ def check_cuda(use_cuda, err = \ sys.exit(1) except Exception as e: pass - -def check_version(): - """ - Log error and exit when the installed version of paddlepaddle is - not satisfied. - """ - err = "PaddlePaddle version 1.6 or higher is required, " \ - "or a suitable develop version is satisfied as well. \n" \ - "Please make sure the version is good with your code." \ - - try: - fluid.require_version('1.6.0') - except Exception as e: - logger.error(err) - sys.exit(1) -- GitLab