diff --git a/configs/dcn/yolov3_enhance_reader.yml b/configs/dcn/yolov3_enhance_reader.yml index 228e5558aa1c616021823aa7f99ac4748a1a9826..6616d6b2a0d7927da9074cbc59eaac4412b44971 100644 --- a/configs/dcn/yolov3_enhance_reader.yml +++ b/configs/dcn/yolov3_enhance_reader.yml @@ -44,7 +44,7 @@ TrainReader: shuffle: true drop_last: true worker_num: 8 - bufsize: 32 + bufsize: 16 use_process: true EvalReader: @@ -76,7 +76,7 @@ EvalReader: batch_size: 8 drop_empty: false worker_num: 8 - bufsize: 32 + bufsize: 16 TestReader: inputs_def: diff --git a/configs/ssd/ssd_mobilenet_v1_voc.yml b/configs/ssd/ssd_mobilenet_v1_voc.yml index 6d727adfbc450ef99a7b97112b50307fef5da281..ec3c5a21c9c29f3fc81da276749259295f10bc7f 100644 --- a/configs/ssd/ssd_mobilenet_v1_voc.yml +++ b/configs/ssd/ssd_mobilenet_v1_voc.yml @@ -116,7 +116,7 @@ EvalReader: std: [127.502231, 127.502231, 127.502231] batch_size: 32 worker_num: 8 - bufsize: 32 + bufsize: 16 use_process: false TestReader: diff --git a/configs/ssd/ssd_vgg16_300.yml b/configs/ssd/ssd_vgg16_300.yml index fed09072dbf03ad6517c2e59f16720bf54f36598..aaacf022dcd61619ef4c4654b3ac1ebeb971bcd9 100644 --- a/configs/ssd/ssd_vgg16_300.yml +++ b/configs/ssd/ssd_vgg16_300.yml @@ -93,7 +93,7 @@ TrainReader: batch_size: 8 shuffle: true worker_num: 8 - bufsize: 32 + bufsize: 16 use_process: true drop_empty: true @@ -123,7 +123,7 @@ EvalReader: std: [1, 1, 1] batch_size: 16 worker_num: 8 - bufsize: 32 + bufsize: 16 TestReader: inputs_def: diff --git a/configs/ssd/ssd_vgg16_300_voc.yml b/configs/ssd/ssd_vgg16_300_voc.yml index 18d6bc3b8012101edd8015e91bb6ad39303c3ba6..b748966887cc58187da06327e5bce737b858cc89 100644 --- a/configs/ssd/ssd_vgg16_300_voc.yml +++ b/configs/ssd/ssd_vgg16_300_voc.yml @@ -93,7 +93,7 @@ TrainReader: batch_size: 8 shuffle: true worker_num: 8 - bufsize: 32 + bufsize: 16 use_process: true EvalReader: @@ -122,7 +122,7 @@ EvalReader: std: [1, 1, 1] batch_size: 32 worker_num: 8 - bufsize: 32 + bufsize: 16 TestReader: inputs_def: diff --git a/configs/ssd/ssd_vgg16_512.yml b/configs/ssd/ssd_vgg16_512.yml index 1383acf4253383524c6e792ee0a366e5d9dbc56d..1ddea55706141a694de02cdc1da2f72af41531cf 100644 --- a/configs/ssd/ssd_vgg16_512.yml +++ b/configs/ssd/ssd_vgg16_512.yml @@ -96,7 +96,7 @@ TrainReader: batch_size: 8 shuffle: true worker_num: 8 - bufsize: 32 + bufsize: 16 use_process: true EvalReader: @@ -124,7 +124,7 @@ EvalReader: std: [1, 1, 1] batch_size: 8 worker_num: 8 - bufsize: 32 + bufsize: 16 drop_empty: false TestReader: diff --git a/configs/ssd/ssd_vgg16_512_voc.yml b/configs/ssd/ssd_vgg16_512_voc.yml index c3224129f1809ecd019aa282c53c71aaca7f8163..35fd19094e366e882e0dd1dd92397f2b2e526c0e 100644 --- a/configs/ssd/ssd_vgg16_512_voc.yml +++ b/configs/ssd/ssd_vgg16_512_voc.yml @@ -97,7 +97,7 @@ TrainReader: batch_size: 8 shuffle: true worker_num: 8 - bufsize: 32 + bufsize: 16 use_process: true EvalReader: @@ -126,7 +126,7 @@ EvalReader: std: [1, 1, 1] batch_size: 32 worker_num: 8 - bufsize: 32 + bufsize: 16 TestReader: inputs_def: diff --git a/configs/yolov3_reader.yml b/configs/yolov3_reader.yml index e539408d13c199c3bd92bce56167f832c7a77d8f..2a8463f1e6c2cb598ea4a55c6289f5b04b290d4a 100644 --- a/configs/yolov3_reader.yml +++ b/configs/yolov3_reader.yml @@ -51,7 +51,7 @@ TrainReader: mixup_epoch: 250 drop_last: true worker_num: 8 - bufsize: 32 + bufsize: 16 use_process: true @@ -84,7 +84,7 @@ EvalReader: batch_size: 8 drop_empty: false worker_num: 8 - bufsize: 32 + bufsize: 16 TestReader: inputs_def: diff --git a/ppdet/data/reader.py b/ppdet/data/reader.py index 04eb749ed1a615d7474bbcfba425bc927193509d..b8608fbaf5638ad19851a29bfeb12693c39308fa 100644 --- a/ppdet/data/reader.py +++ b/ppdet/data/reader.py @@ -179,6 +179,7 @@ class Reader(object): use_process is true. Default 3G. inputs_def (dict): network input definition use to get input fields, which is used to determine the order of returned data. + devices_num (int): number of devices. """ def __init__(self, @@ -195,9 +196,10 @@ class Reader(object): use_process=False, use_fine_grained_loss=False, num_classes=80, - bufsize=100, + bufsize=-1, memsize='3G', - inputs_def=None): + inputs_def=None, + devices_num=1): self._dataset = dataset self._roidbs = self._dataset.get_roidb() self._fields = copy.deepcopy(inputs_def[ @@ -256,6 +258,7 @@ class Reader(object): self._parallel = None if self._worker_num > -1: task = functools.partial(self.worker, self._drop_empty) + bufsize = devices_num * 2 if bufsize == -1 else bufsize self._parallel = ParallelMap(self, task, worker_num, bufsize, use_process, memsize) @@ -388,7 +391,7 @@ class Reader(object): self._parallel.stop() -def create_reader(cfg, max_iter=0, global_cfg=None): +def create_reader(cfg, max_iter=0, global_cfg=None, devices_num=1): """ Return iterable data reader. @@ -403,6 +406,7 @@ def create_reader(cfg, max_iter=0, global_cfg=None): cfg['use_fine_grained_loss'] = getattr(global_cfg, 'use_fine_grained_loss', False) cfg['num_classes'] = getattr(global_cfg, 'num_classes', 80) + cfg['devices_num'] = devices_num reader = Reader(**cfg)() def _reader(): diff --git a/ppdet/modeling/architectures/blazeface.py b/ppdet/modeling/architectures/blazeface.py index b8a46535e901a12fa0c23c5e51d2978d6b32ccf6..245a43835fcda15ef541ae7c42f2163042ade411 100644 --- a/ppdet/modeling/architectures/blazeface.py +++ b/ppdet/modeling/architectures/blazeface.py @@ -198,7 +198,7 @@ class BlazeFace(object): lod_level=inputs_def[key]['lod_level'])) for key in fields]) loader = fluid.io.DataLoader.from_generator( feed_list=list(feed_vars.values()), - capacity=64, + capacity=16, use_double_buffer=True, iterable=iterable) if use_dataloader else None return feed_vars, loader diff --git a/ppdet/modeling/architectures/cascade_mask_rcnn.py b/ppdet/modeling/architectures/cascade_mask_rcnn.py index c99cfd1c15f5c54e56a4273cb18f8573d93d410d..30180ddacfe1742a82fa147f9c9e727a36135e79 100644 --- a/ppdet/modeling/architectures/cascade_mask_rcnn.py +++ b/ppdet/modeling/architectures/cascade_mask_rcnn.py @@ -421,7 +421,7 @@ class CascadeMaskRCNN(object): use_dataloader = use_dataloader and not mask_branch loader = fluid.io.DataLoader.from_generator( feed_list=list(feed_vars.values()), - capacity=64, + capacity=16, use_double_buffer=True, iterable=iterable) if use_dataloader else None return feed_vars, loader diff --git a/ppdet/modeling/architectures/cascade_rcnn.py b/ppdet/modeling/architectures/cascade_rcnn.py index 60e15ad1c87c542227ddf265c8bb50ae42b3e3ee..1e27888cf7cef9e342f59b474e692b414f45c98d 100644 --- a/ppdet/modeling/architectures/cascade_rcnn.py +++ b/ppdet/modeling/architectures/cascade_rcnn.py @@ -318,7 +318,7 @@ class CascadeRCNN(object): lod_level=inputs_def[key]['lod_level'])) for key in fields]) loader = fluid.io.DataLoader.from_generator( feed_list=list(feed_vars.values()), - capacity=64, + capacity=16, use_double_buffer=True, iterable=iterable) if use_dataloader else None return feed_vars, loader diff --git a/ppdet/modeling/architectures/cascade_rcnn_cls_aware.py b/ppdet/modeling/architectures/cascade_rcnn_cls_aware.py index 364007ba8ea7c737361a52e3b602473666101a09..1af96e38d1ce1ad54ee5b5f2601fc958dd7aa524 100644 --- a/ppdet/modeling/architectures/cascade_rcnn_cls_aware.py +++ b/ppdet/modeling/architectures/cascade_rcnn_cls_aware.py @@ -301,7 +301,7 @@ class CascadeRCNNClsAware(object): lod_level=inputs_def[key]['lod_level'])) for key in fields]) loader = fluid.io.DataLoader.from_generator( feed_list=list(feed_vars.values()), - capacity=64, + capacity=16, use_double_buffer=True, iterable=iterable) if use_dataloader else None return feed_vars, loader diff --git a/ppdet/modeling/architectures/faceboxes.py b/ppdet/modeling/architectures/faceboxes.py index ee7292017a2afa0933c79cb5a0a92e731f763f36..f5b120ecef430973d95053742cea282fbb729f34 100644 --- a/ppdet/modeling/architectures/faceboxes.py +++ b/ppdet/modeling/architectures/faceboxes.py @@ -172,7 +172,7 @@ class FaceBoxes(object): lod_level=inputs_def[key]['lod_level'])) for key in fields]) loader = fluid.io.DataLoader.from_generator( feed_list=list(feed_vars.values()), - capacity=64, + capacity=16, use_double_buffer=True, iterable=iterable) if use_dataloader else None return feed_vars, loader diff --git a/ppdet/modeling/architectures/faster_rcnn.py b/ppdet/modeling/architectures/faster_rcnn.py index 239af60780d06441c7a18183f50fc2b78003b8c4..1b22ad67f9493488a6a00f09251437e20644bb6b 100644 --- a/ppdet/modeling/architectures/faster_rcnn.py +++ b/ppdet/modeling/architectures/faster_rcnn.py @@ -231,7 +231,7 @@ class FasterRCNN(object): lod_level=inputs_def[key]['lod_level'])) for key in fields]) loader = fluid.io.DataLoader.from_generator( feed_list=list(feed_vars.values()), - capacity=64, + capacity=16, use_double_buffer=True, iterable=iterable) if use_dataloader else None return feed_vars, loader diff --git a/ppdet/modeling/architectures/mask_rcnn.py b/ppdet/modeling/architectures/mask_rcnn.py index 5581900090150704025a00da505281e549aeacda..1f3f0104a14e023ba44c50e514eabe4f4c39b92c 100644 --- a/ppdet/modeling/architectures/mask_rcnn.py +++ b/ppdet/modeling/architectures/mask_rcnn.py @@ -324,7 +324,7 @@ class MaskRCNN(object): use_dataloader = use_dataloader and not mask_branch loader = fluid.io.DataLoader.from_generator( feed_list=list(feed_vars.values()), - capacity=64, + capacity=16, use_double_buffer=True, iterable=iterable) if use_dataloader else None return feed_vars, loader diff --git a/ppdet/modeling/architectures/retinanet.py b/ppdet/modeling/architectures/retinanet.py index 936a4ba4f9b8bf003186e0e2862d52e4d7c6149b..c541bc2b2a7ace00a98d2b72608dc274d237b3ce 100644 --- a/ppdet/modeling/architectures/retinanet.py +++ b/ppdet/modeling/architectures/retinanet.py @@ -114,7 +114,7 @@ class RetinaNet(object): lod_level=inputs_def[key]['lod_level'])) for key in fields]) loader = fluid.io.DataLoader.from_generator( feed_list=list(feed_vars.values()), - capacity=64, + capacity=16, use_double_buffer=True, iterable=iterable) if use_dataloader else None return feed_vars, loader diff --git a/ppdet/modeling/architectures/ssd.py b/ppdet/modeling/architectures/ssd.py index 70aaf7319e259195e58abe325b00bb2eae6fad2b..5e082479d87ca3ae64ed68d658fd26e9b6ad386e 100644 --- a/ppdet/modeling/architectures/ssd.py +++ b/ppdet/modeling/architectures/ssd.py @@ -118,7 +118,7 @@ class SSD(object): lod_level=inputs_def[key]['lod_level'])) for key in fields]) loader = fluid.io.DataLoader.from_generator( feed_list=list(feed_vars.values()), - capacity=64, + capacity=16, use_double_buffer=True, iterable=iterable) if use_dataloader else None return feed_vars, loader diff --git a/ppdet/modeling/architectures/yolov3.py b/ppdet/modeling/architectures/yolov3.py index cff0dec6df58dad0e7ca9f3e4991bb5fb4d99309..52bb41932606afa849993568919176d6768fa749 100644 --- a/ppdet/modeling/architectures/yolov3.py +++ b/ppdet/modeling/architectures/yolov3.py @@ -147,7 +147,7 @@ class YOLOv3(object): lod_level=inputs_def[key]['lod_level'])) for key in fields]) loader = fluid.io.DataLoader.from_generator( feed_list=list(feed_vars.values()), - capacity=64, + capacity=16, use_double_buffer=True, iterable=iterable) if use_dataloader else None return feed_vars, loader diff --git a/tools/eval.py b/tools/eval.py index 7eb3f18f3ac807600f911b531d7101cbc97c2599..2feb5150fbb1c788cf743f63abde7f4c09b91174 100644 --- a/tools/eval.py +++ b/tools/eval.py @@ -84,7 +84,7 @@ def main(): fetches = model.eval(feed_vars, multi_scale_test) eval_prog = eval_prog.clone(True) - reader = create_reader(cfg.EvalReader) + reader = create_reader(cfg.EvalReader, devices_num=1) loader.set_sample_list_generator(reader, place) dataset = cfg['EvalReader']['dataset'] diff --git a/tools/infer.py b/tools/infer.py index 3835264b2099562e4dfcb3accf07ebf02119b886..a81413b820ec9457e64b8d6f568d2bbd58fd3455 100644 --- a/tools/infer.py +++ b/tools/infer.py @@ -130,7 +130,7 @@ def main(): test_fetches = model.test(feed_vars) infer_prog = infer_prog.clone(True) - reader = create_reader(cfg.TestReader) + reader = create_reader(cfg.TestReader, devices_num=1) loader.set_sample_list_generator(reader, place) exe.run(startup_prog) diff --git a/tools/train.py b/tools/train.py index 846da330abfb2a4142509ac0b3384bb9262916e6..54af54ab809348ed2c7b91c1f622a5bd24a34cc7 100644 --- a/tools/train.py +++ b/tools/train.py @@ -144,7 +144,7 @@ def main(): fetches = model.eval(feed_vars) eval_prog = eval_prog.clone(True) - eval_reader = create_reader(cfg.EvalReader) + eval_reader = create_reader(cfg.EvalReader, devices_num=1) eval_loader.set_sample_list_generator(eval_reader, place) # parse eval fetches @@ -200,8 +200,10 @@ def main(): checkpoint.load_params( exe, train_prog, cfg.pretrain_weights, ignore_params=ignore_params) - train_reader = create_reader(cfg.TrainReader, (cfg.max_iters - start_iter) * - devices_num, cfg) + train_reader = create_reader( + cfg.TrainReader, (cfg.max_iters - start_iter) * devices_num, + cfg, + devices_num=devices_num) train_loader.set_sample_list_generator(train_reader, place) # whether output bbox is normalized in model output layer