Cherry pick fix buffsize (#475)

* add devices_num on reader

* refine capacity value

* refine bufsize

configs/dcn/yolov3_enhance_reader.yml

@@ -44,7 +44,7 @@ TrainReader:
   shuffle: true
   drop_last: true
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
   use_process: true
 
 EvalReader:
@@ -78,7 +78,7 @@ EvalReader:
   batch_size: 8
   drop_empty: false
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
 
 TestReader:
   inputs_def:

configs/ssd/ssd_mobilenet_v1_voc.yml

@@ -116,7 +116,7 @@ EvalReader:
     std: [127.502231, 127.502231, 127.502231]
   batch_size: 32
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
   use_process: false
 
 TestReader:

configs/ssd/ssd_vgg16_300.yml

@@ -93,7 +93,7 @@ TrainReader:
   batch_size: 8
   shuffle: true
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
   use_process: true
   drop_empty: true
 
@@ -123,7 +123,7 @@ EvalReader:
     std: [1, 1, 1]
   batch_size: 16
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
 
 TestReader:
   inputs_def:

configs/ssd/ssd_vgg16_300_voc.yml

@@ -93,7 +93,7 @@ TrainReader:
   batch_size: 8
   shuffle: true
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
   use_process: true
 
 EvalReader:
@@ -122,7 +122,7 @@ EvalReader:
     std: [1, 1, 1]
   batch_size: 32
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
 
 TestReader:
   inputs_def:

configs/ssd/ssd_vgg16_512.yml

@@ -96,7 +96,7 @@ TrainReader:
   batch_size: 8
   shuffle: true
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
   use_process: true
 
 EvalReader:
@@ -124,7 +124,7 @@ EvalReader:
     std: [1, 1, 1]
   batch_size: 8
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
   drop_empty: false
 
 TestReader:

configs/ssd/ssd_vgg16_512_voc.yml

@@ -97,7 +97,7 @@ TrainReader:
   batch_size: 8
   shuffle: true
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
   use_process: true
 
 EvalReader:
@@ -126,7 +126,7 @@ EvalReader:
     std: [1, 1, 1]
   batch_size: 32
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
 
 TestReader:
   inputs_def:

configs/yolov3_reader.yml

@@ -51,7 +51,7 @@ TrainReader:
   mixup_epoch: 250
   drop_last: true
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
   use_process: true
 
 
@@ -84,7 +84,7 @@ EvalReader:
   batch_size: 8
   drop_empty: false
   worker_num: 8
-  bufsize: 32
+  bufsize: 16
 
 TestReader:
   inputs_def:

ppdet/data/reader.py

@@ -179,6 +179,7 @@ class Reader(object):
             use_process is true. Default 3G.
         inputs_def (dict): network input definition use to get input fields,
             which is used to determine the order of returned data.
+        devices_num (int): number of devices.
     """
 
     def __init__(self,
@@ -195,9 +196,10 @@ class Reader(object):
                  use_process=False,
                  use_fine_grained_loss=False,
                  num_classes=80,
-                 bufsize=100,
+                 bufsize=-1,
                  memsize='3G',
-                 inputs_def=None):
+                 inputs_def=None,
+                 devices_num=1):
         self._dataset = dataset
         self._roidbs = self._dataset.get_roidb()
         self._fields = copy.deepcopy(inputs_def[
@@ -256,6 +258,7 @@ class Reader(object):
         self._parallel = None
         if self._worker_num > -1:
             task = functools.partial(self.worker, self._drop_empty)
+            bufsize = devices_num * 2 if bufsize == -1 else bufsize
             self._parallel = ParallelMap(self, task, worker_num, bufsize,
                                          use_process, memsize)
 
@@ -388,7 +391,7 @@ class Reader(object):
             self._parallel.stop()
 
 
-def create_reader(cfg, max_iter=0, global_cfg=None):
+def create_reader(cfg, max_iter=0, global_cfg=None, devices_num=1):
     """
     Return iterable data reader.
 
@@ -403,6 +406,7 @@ def create_reader(cfg, max_iter=0, global_cfg=None):
         cfg['use_fine_grained_loss'] = getattr(global_cfg,
                                                'use_fine_grained_loss', False)
         cfg['num_classes'] = getattr(global_cfg, 'num_classes', 80)
+    cfg['devices_num'] = devices_num
     reader = Reader(**cfg)()
 
     def _reader():

ppdet/modeling/architectures/blazeface.py

@@ -198,7 +198,7 @@ class BlazeFace(object):
             lod_level=inputs_def[key]['lod_level'])) for key in fields])
         loader = fluid.io.DataLoader.from_generator(
             feed_list=list(feed_vars.values()),
-            capacity=64,
+            capacity=16,
             use_double_buffer=True,
             iterable=iterable) if use_dataloader else None
         return feed_vars, loader

ppdet/modeling/architectures/cascade_mask_rcnn.py

@@ -421,7 +421,7 @@ class CascadeMaskRCNN(object):
         use_dataloader = use_dataloader and not mask_branch
         loader = fluid.io.DataLoader.from_generator(
             feed_list=list(feed_vars.values()),
-            capacity=64,
+            capacity=16,
             use_double_buffer=True,
             iterable=iterable) if use_dataloader else None
         return feed_vars, loader

ppdet/modeling/architectures/cascade_rcnn.py

@@ -319,7 +319,7 @@ class CascadeRCNN(object):
             lod_level=inputs_def[key]['lod_level'])) for key in fields])
         loader = fluid.io.DataLoader.from_generator(
             feed_list=list(feed_vars.values()),
-            capacity=64,
+            capacity=16,
             use_double_buffer=True,
             iterable=iterable) if use_dataloader else None
         return feed_vars, loader

ppdet/modeling/architectures/cascade_rcnn_cls_aware.py

@@ -202,7 +202,7 @@ class CascadeRCNNClsAware(object):
             lod_level=inputs_def[key]['lod_level'])) for key in fields])
         loader = fluid.io.DataLoader.from_generator(
             feed_list=list(feed_vars.values()),
-            capacity=64,
+            capacity=16,
             use_double_buffer=True,
             iterable=iterable) if use_dataloader else None
         return feed_vars, loader

ppdet/modeling/architectures/faceboxes.py

@@ -172,7 +172,7 @@ class FaceBoxes(object):
             lod_level=inputs_def[key]['lod_level'])) for key in fields])
         loader = fluid.io.DataLoader.from_generator(
             feed_list=list(feed_vars.values()),
-            capacity=64,
+            capacity=16,
             use_double_buffer=True,
             iterable=iterable) if use_dataloader else None
         return feed_vars, loader

ppdet/modeling/architectures/faster_rcnn.py

@@ -231,7 +231,7 @@ class FasterRCNN(object):
             lod_level=inputs_def[key]['lod_level'])) for key in fields])
         loader = fluid.io.DataLoader.from_generator(
             feed_list=list(feed_vars.values()),
-            capacity=64,
+            capacity=16,
             use_double_buffer=True,
             iterable=iterable) if use_dataloader else None
         return feed_vars, loader

ppdet/modeling/architectures/mask_rcnn.py

@@ -324,7 +324,7 @@ class MaskRCNN(object):
         use_dataloader = use_dataloader and not mask_branch
         loader = fluid.io.DataLoader.from_generator(
             feed_list=list(feed_vars.values()),
-            capacity=64,
+            capacity=16,
             use_double_buffer=True,
             iterable=iterable) if use_dataloader else None
         return feed_vars, loader

ppdet/modeling/architectures/retinanet.py

@@ -114,7 +114,7 @@ class RetinaNet(object):
             lod_level=inputs_def[key]['lod_level'])) for key in fields])
         loader = fluid.io.DataLoader.from_generator(
             feed_list=list(feed_vars.values()),
-            capacity=64,
+            capacity=16,
             use_double_buffer=True,
             iterable=iterable) if use_dataloader else None
         return feed_vars, loader

ppdet/modeling/architectures/ssd.py

@@ -118,7 +118,7 @@ class SSD(object):
             lod_level=inputs_def[key]['lod_level'])) for key in fields])
         loader = fluid.io.DataLoader.from_generator(
             feed_list=list(feed_vars.values()),
-            capacity=64,
+            capacity=16,
             use_double_buffer=True,
             iterable=iterable) if use_dataloader else None
         return feed_vars, loader

ppdet/modeling/architectures/yolov3.py

@@ -147,7 +147,7 @@ class YOLOv3(object):
             lod_level=inputs_def[key]['lod_level'])) for key in fields])
         loader = fluid.io.DataLoader.from_generator(
             feed_list=list(feed_vars.values()),
-            capacity=64,
+            capacity=16,
             use_double_buffer=True,
             iterable=iterable) if use_dataloader else None
         return feed_vars, loader

tools/eval.py

@@ -84,7 +84,7 @@ def main():
                 fetches = model.eval(feed_vars, multi_scale_test)
     eval_prog = eval_prog.clone(True)
 
-    reader = create_reader(cfg.EvalReader)
+    reader = create_reader(cfg.EvalReader, devices_num=1)
     loader.set_sample_list_generator(reader, place)
 
     dataset = cfg['EvalReader']['dataset']

tools/infer.py

@@ -130,7 +130,7 @@ def main():
             test_fetches = model.test(feed_vars)
     infer_prog = infer_prog.clone(True)
 
-    reader = create_reader(cfg.TestReader)
+    reader = create_reader(cfg.TestReader, devices_num=1)
     loader.set_sample_list_generator(reader, place)
 
     exe.run(startup_prog)

tools/train.py

@@ -137,7 +137,7 @@ def main():
                 fetches = model.eval(feed_vars)
         eval_prog = eval_prog.clone(True)
 
-        eval_reader = create_reader(cfg.EvalReader)
+        eval_reader = create_reader(cfg.EvalReader, devices_num=1)
         eval_loader.set_sample_list_generator(eval_reader, place)
 
         # parse eval fetches
@@ -193,8 +193,10 @@ def main():
         checkpoint.load_params(
             exe, train_prog, cfg.pretrain_weights, ignore_params=ignore_params)
 
-    train_reader = create_reader(cfg.TrainReader, (cfg.max_iters - start_iter) *
-                                 devices_num, cfg)
+    train_reader = create_reader(
+        cfg.TrainReader, (cfg.max_iters - start_iter) * devices_num,
+        cfg,
+        devices_num=devices_num)
     train_loader.set_sample_list_generator(train_reader, place)
 
     # whether output bbox is normalized in model output layer