From 0d1721e6ae13b70e1aff342165ff6ee7ca499166 Mon Sep 17 00:00:00 2001 From: still-wait <742925032@qq.com> Date: Thu, 24 Sep 2020 06:55:31 +0000 Subject: [PATCH] clean useless log in DataLoader --- .gitignore | 3 +++ docs/advanced_tutorials/READER.md | 2 +- slim/distillation/distill.py | 6 ++++-- .../distill_pruned_model/distill_pruned_model.py | 6 ++++-- slim/nas/train_nas.py | 7 ++++--- slim/prune/eval.py | 3 ++- slim/prune/infer.py | 3 ++- slim/prune/prune.py | 3 ++- slim/quantization/eval.py | 3 ++- slim/quantization/infer.py | 3 ++- slim/quantization/train.py | 6 ++++-- slim/sensitive/sensitive.py | 3 ++- tools/eval.py | 3 ++- tools/infer.py | 3 ++- tools/train.py | 6 ++++-- tools/train_multi_machine.py | 6 ++++-- 16 files changed, 44 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 4c39d631f..d91c22d16 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,9 @@ __pycache__/ # json file *.json +# log file +*.log + # Distribution / packaging /bin/ /build/ diff --git a/docs/advanced_tutorials/READER.md b/docs/advanced_tutorials/READER.md index 836ef0a5c..d809ad82a 100644 --- a/docs/advanced_tutorials/READER.md +++ b/docs/advanced_tutorials/READER.md @@ -406,7 +406,7 @@ reader = create_reader(cfg.EvalReader) # infer reader = create_reader(cfg.TestReader) # 将reader设置为DataLoader数据源 -loader.set_sample_list_generator(reader, place) +loader.set_sample_list_generator(reader) ``` 在运行程序中设置完数据处理模块后，就可以开始训练、评估与测试了，具体请参考相应运行程序python源码。 diff --git a/slim/distillation/distill.py b/slim/distillation/distill.py index dd3a86c0f..0372914bf 100644 --- a/slim/distillation/distill.py +++ b/slim/distillation/distill.py @@ -160,7 +160,8 @@ def main(): start_iter = 0 train_reader = create_reader(cfg.TrainReader, (cfg.max_iters - start_iter) * devices_num, cfg) - train_loader.set_sample_list_generator(train_reader, place) + # When iterable mode, set set_sample_list_generator(train_reader, place) + train_loader.set_sample_list_generator(train_reader) # get all student variables student_vars = [] @@ -183,7 +184,8 @@ def main(): eval_prog = eval_prog.clone(True) eval_reader = create_reader(cfg.EvalReader) - eval_loader.set_sample_list_generator(eval_reader, place) + # When iterable mode, set set_sample_list_generator(eval_reader, place) + eval_loader.set_sample_list_generator(eval_reader) # parse eval fetches extra_keys = [] diff --git a/slim/extensions/distill_pruned_model/distill_pruned_model.py b/slim/extensions/distill_pruned_model/distill_pruned_model.py index a2adc6cf7..573322061 100644 --- a/slim/extensions/distill_pruned_model/distill_pruned_model.py +++ b/slim/extensions/distill_pruned_model/distill_pruned_model.py @@ -149,7 +149,8 @@ def main(): start_iter = 0 train_reader = create_reader(cfg.TrainReader, (cfg.max_iters - start_iter) * devices_num, cfg) - train_loader.set_sample_list_generator(train_reader, place) + # When iterable mode, set set_sample_list_generator(train_reader, place) + train_loader.set_sample_list_generator(train_reader) eval_prog = fluid.Program() with fluid.program_guard(eval_prog, fluid.default_startup_program()): @@ -161,7 +162,8 @@ def main(): eval_prog = eval_prog.clone(True) eval_reader = create_reader(cfg.EvalReader) - eval_loader.set_sample_list_generator(eval_reader, place) + # When iterable mode, set set_sample_list_generator(eval_reader, place) + eval_loader.set_sample_list_generator(eval_reader) teacher_cfg = load_config(FLAGS.teacher_config) merge_config(FLAGS.opt) diff --git a/slim/nas/train_nas.py b/slim/nas/train_nas.py index 3c30c7ffd..07bd2b58a 100644 --- a/slim/nas/train_nas.py +++ b/slim/nas/train_nas.py @@ -296,7 +296,8 @@ def main(): fetches = archs(feed_vars, 'eval', cfg) eval_prog = eval_prog.clone(True) - eval_loader.set_sample_list_generator(eval_reader, place) + # When iterable mode, set set_sample_list_generator(eval_reader, place) + eval_loader.set_sample_list_generator(eval_reader) extra_keys = ['im_id', 'im_shape', 'gt_bbox'] eval_keys, eval_values, eval_cls = parse_fetches(fetches, eval_prog, extra_keys) @@ -324,8 +325,8 @@ def main(): exec_strategy=exec_strategy) if FLAGS.eval: compiled_eval_prog = fluid.CompiledProgram(eval_prog) - - train_loader.set_sample_list_generator(train_reader, place) + # When iterable mode, set set_sample_list_generator(train_reader, place) + train_loader.set_sample_list_generator(train_reader) train_stats = TrainingStats(cfg.log_smooth_window, train_keys) train_loader.start() diff --git a/slim/prune/eval.py b/slim/prune/eval.py index 73386d0f7..82281fea6 100644 --- a/slim/prune/eval.py +++ b/slim/prune/eval.py @@ -78,7 +78,8 @@ def main(): exe.run(startup_prog) reader = create_reader(cfg.EvalReader) - loader.set_sample_list_generator(reader, place) + # When iterable mode, set set_sample_list_generator(reader, place) + loader.set_sample_list_generator(reader) dataset = cfg['EvalReader']['dataset'] diff --git a/slim/prune/infer.py b/slim/prune/infer.py index 7c7d82592..ed2e2a100 100644 --- a/slim/prune/infer.py +++ b/slim/prune/infer.py @@ -147,7 +147,8 @@ def main(): logger.info("pruned FLOPS: {}".format( float(base_flops - pruned_flops) / base_flops)) reader = create_reader(cfg.TestReader, devices_num=1) - loader.set_sample_list_generator(reader, place) + # When iterable mode, set set_sample_list_generator(reader, place) + loader.set_sample_list_generator(reader) exe.run(startup_prog) if cfg.weights: diff --git a/slim/prune/prune.py b/slim/prune/prune.py index 01b3488d7..795905bee 100644 --- a/slim/prune/prune.py +++ b/slim/prune/prune.py @@ -132,7 +132,8 @@ def main(): eval_prog = eval_prog.clone(True) eval_reader = create_reader(cfg.EvalReader) - eval_loader.set_sample_list_generator(eval_reader, place) + # When iterable mode, set set_sample_list_generator(eval_reader, place) + eval_loader.set_sample_list_generator(eval_reader) # parse eval fetches extra_keys = [] diff --git a/slim/quantization/eval.py b/slim/quantization/eval.py index ecc77e45f..cd5216a25 100644 --- a/slim/quantization/eval.py +++ b/slim/quantization/eval.py @@ -73,7 +73,8 @@ def main(): eval_prog = eval_prog.clone(True) reader = create_reader(cfg.EvalReader) - loader.set_sample_list_generator(reader, place) + # When iterable mode, set set_sample_list_generator(reader, place) + loader.set_sample_list_generator(reader) # eval already exists json file if FLAGS.json_eval: diff --git a/slim/quantization/infer.py b/slim/quantization/infer.py index cb16d3b95..a27b804c2 100644 --- a/slim/quantization/infer.py +++ b/slim/quantization/infer.py @@ -75,7 +75,8 @@ def main(): infer_prog = infer_prog.clone(True) reader = create_reader(cfg.TestReader) - loader.set_sample_list_generator(reader, place) + # When iterable mode, set set_sample_list_generator(reader, place) + loader.set_sample_list_generator(reader) not_quant_pattern = [] if FLAGS.not_quant_pattern: not_quant_pattern = FLAGS.not_quant_pattern diff --git a/slim/quantization/train.py b/slim/quantization/train.py index 63c65816b..81fe62e0c 100644 --- a/slim/quantization/train.py +++ b/slim/quantization/train.py @@ -129,7 +129,8 @@ def main(): eval_prog = eval_prog.clone(True) eval_reader = create_reader(cfg.EvalReader) - eval_loader.set_sample_list_generator(eval_reader, place) + # When iterable mode, set set_sample_list_generator(eval_reader, place) + eval_loader.set_sample_list_generator(eval_reader) # parse eval fetches extra_keys = [] @@ -210,7 +211,8 @@ def main(): train_reader = create_reader(cfg.TrainReader, (cfg.max_iters - start_iter) * devices_num) - train_loader.set_sample_list_generator(train_reader, place) + # When iterable mode, set set_sample_list_generator(train_reader, place) + train_loader.set_sample_list_generator(train_reader) # whether output bbox is normalized in model output layer is_bbox_normalized = False diff --git a/slim/sensitive/sensitive.py b/slim/sensitive/sensitive.py index 4ddea2015..a825f199b 100644 --- a/slim/sensitive/sensitive.py +++ b/slim/sensitive/sensitive.py @@ -84,7 +84,8 @@ def main(): return eval_reader = create_reader(cfg.EvalReader) - eval_loader.set_sample_list_generator(eval_reader, place) + # When iterable mode, set set_sample_list_generator(eval_reader, place) + eval_loader.set_sample_list_generator(eval_reader) # parse eval fetches extra_keys = [] diff --git a/tools/eval.py b/tools/eval.py index 84da9b49a..e7daf2bce 100644 --- a/tools/eval.py +++ b/tools/eval.py @@ -74,7 +74,8 @@ def main(): eval_prog = eval_prog.clone(True) reader = create_reader(cfg.EvalReader, devices_num=1) - loader.set_sample_list_generator(reader, place) + # When iterable mode, set set_sample_list_generator(reader, place) + loader.set_sample_list_generator(reader) dataset = cfg['EvalReader']['dataset'] diff --git a/tools/infer.py b/tools/infer.py index 034e33e66..4f2ae5743 100644 --- a/tools/infer.py +++ b/tools/infer.py @@ -120,7 +120,8 @@ def main(): infer_prog = infer_prog.clone(True) reader = create_reader(cfg.TestReader, devices_num=1) - loader.set_sample_list_generator(reader, place) + # When iterable mode, set set_sample_list_generator(reader, place) + loader.set_sample_list_generator(reader) exe.run(startup_prog) if cfg.weights: diff --git a/tools/train.py b/tools/train.py index dd2edbd43..0a541667d 100644 --- a/tools/train.py +++ b/tools/train.py @@ -146,7 +146,8 @@ def main(): eval_prog = eval_prog.clone(True) eval_reader = create_reader(cfg.EvalReader, devices_num=1) - eval_loader.set_sample_list_generator(eval_reader, place) + # When iterable mode, set set_sample_list_generator(eval_reader, place) + eval_loader.set_sample_list_generator(eval_reader) # parse eval fetches extra_keys = [] @@ -206,7 +207,8 @@ def main(): cfg, devices_num=devices_num, num_trainers=num_trainers) - train_loader.set_sample_list_generator(train_reader, place) + # When iterable mode, set set_sample_list_generator(train_reader, place) + train_loader.set_sample_list_generator(train_reader) # whether output bbox is normalized in model output layer is_bbox_normalized = False diff --git a/tools/train_multi_machine.py b/tools/train_multi_machine.py index 2a82f0505..6cfa8be7c 100644 --- a/tools/train_multi_machine.py +++ b/tools/train_multi_machine.py @@ -162,7 +162,8 @@ def main(): eval_prog = eval_prog.clone(True) eval_reader = create_reader(cfg.EvalReader, devices_num=1) - eval_loader.set_sample_list_generator(eval_reader, place) + # When iterable mode, set set_sample_list_generator(eval_reader, place) + eval_loader.set_sample_list_generator(eval_reader) # parse eval fetches extra_keys = [] @@ -200,7 +201,8 @@ def main(): cfg.TrainReader, (cfg.max_iters - start_iter) * devices_num, cfg, devices_num=devices_num) - train_loader.set_sample_list_generator(train_reader, place) + # When iterable mode, set set_sample_list_generator(train_reader, place) + train_loader.set_sample_list_generator(train_reader) # whether output bbox is normalized in model output layer is_bbox_normalized = False -- GitLab