diff --git a/ppcls/utils/config.py b/ppcls/utils/config.py index e1712a8a83768f83e6e44e3bdf7fdf8652ffed76..54ee51752a6658972a93a88380556963698c9cf4 100644 --- a/ppcls/utils/config.py +++ b/ppcls/utils/config.py @@ -144,9 +144,14 @@ def override(dl, ks, v): override(dl[k], ks[1:], v) else: if len(ks) == 1: - assert ks[0] in dl, ('{} is not exist in {}'.format(ks[0], dl)) + #assert ks[0] in dl, ('{} is not exist in {}'.format(ks[0], dl)) + if not ks[0] in dl: + logger.warning('A new filed ({}) detected!'.format(ks[0], dl)) dl[ks[0]] = str2num(v) else: + assert ks[0] in dl, ( + '({}) doesn\'t exist in {}, a new dict field is invalid'. + format(ks[0], dl)) override(dl[ks[0]], ks[1:], v) diff --git a/tools/eval.py b/tools/eval.py index 291f77f05d1361cc0bc75ebbe23afcff15987005..7aa370c62bf7ac271f30c955a4b1978a93e51586 100644 --- a/tools/eval.py +++ b/tools/eval.py @@ -74,7 +74,7 @@ def main(args): compiled_valid_prog = program.compile(config, valid_prog) program.run(valid_dataloader, exe, compiled_valid_prog, valid_fetchs, -1, - 'eval') + 'eval', config) if __name__ == '__main__': diff --git a/tools/program.py b/tools/program.py index 4b27e89ad33464ccd6debdba442bfee714ac5044..12a1a6eda5e587e4a5b87f157bc69c9446ed1acd 100644 --- a/tools/program.py +++ b/tools/program.py @@ -410,6 +410,7 @@ def run(dataloader, fetchs, epoch=0, mode='train', + config=None, vdl_writer=None): """ Feed data to the model and fetch the measures and loss @@ -443,16 +444,28 @@ def run(dataloader, logger.scaler('loss', metrics[0][0], total_step, vdl_writer) total_step += 1 if mode == 'eval': - logger.info("{:s} step:{:<4d} {:s}s".format(mode, idx, fetchs_str)) + if idx % config.get('print_interval', 10) == 0: + logger.info("{:s} step:{:<4d} {:s}".format(mode, idx, + fetchs_str)) else: epoch_str = "epoch:{:<3d}".format(epoch) step_str = "{:s} step:{:<4d}".format(mode, idx) - logger.info("{:s} {:s} {:s}".format( - logger.coloring(epoch_str, "HEADER") - if idx == 0 else epoch_str, - logger.coloring(step_str, "PURPLE"), - logger.coloring(fetchs_str, 'OKGREEN'))) + # Keep the first 10 batches statistics, They are important for develop + if epoch == 0 and idx < 10: + logger.info("{:s} {:s} {:s}".format( + logger.coloring(epoch_str, "HEADER") + if idx == 0 else epoch_str, + logger.coloring(step_str, "PURPLE"), + logger.coloring(fetchs_str, 'OKGREEN'))) + + else: + if idx % config.get('print_interval', 10) == 0: + logger.info("{:s} {:s} {:s}".format( + logger.coloring(epoch_str, "HEADER") + if idx == 0 else epoch_str, + logger.coloring(step_str, "PURPLE"), + logger.coloring(fetchs_str, 'OKGREEN'))) end_str = ''.join([str(m.mean) + ' ' for m in metric_list] + [batch_time.total]) + 's' diff --git a/tools/run.sh b/tools/run.sh index 55f2918d91387a88227027656e9ac5e00320d358..5e8043b1205f99dada7448d172f84cd0661df509 100755 --- a/tools/run.sh +++ b/tools/run.sh @@ -5,4 +5,5 @@ export PYTHONPATH=$PWD:$PYTHONPATH python -m paddle.distributed.launch \ --selected_gpus="0,1,2,3" \ tools/train.py \ - -c ./configs/ResNet/ResNet50.yaml + -c ./configs/ResNet/ResNet50.yaml \ + -o print_interval=10 diff --git a/tools/train.py b/tools/train.py index 3456e2aec4dd6f50547bbda927971a6b309cc663..e9188eb0d343b576ee1ad936553b914aa6e75d8f 100644 --- a/tools/train.py +++ b/tools/train.py @@ -110,21 +110,21 @@ def main(args): for epoch_id in range(config.epochs): # 1. train with train dataset program.run(train_dataloader, exe, compiled_train_prog, train_fetchs, - epoch_id, 'train', vdl_writer) + epoch_id, 'train', config, vdl_writer) if int(os.getenv("PADDLE_TRAINER_ID", 0)) == 0: # 2. validate with validate dataset if config.validate and epoch_id % config.valid_interval == 0: if config.get('use_ema'): logger.info(logger.coloring("EMA validate start...")) with ema.apply(exe): - top1_acc = program.run(valid_dataloader, exe, - compiled_valid_prog, - valid_fetchs, epoch_id, 'valid') + top1_acc = program.run( + valid_dataloader, exe, compiled_valid_prog, + valid_fetchs, epoch_id, 'valid', config) logger.info(logger.coloring("EMA validate over!")) top1_acc = program.run(valid_dataloader, exe, compiled_valid_prog, valid_fetchs, - epoch_id, 'valid') + epoch_id, 'valid', config) if top1_acc > best_top1_acc: best_top1_acc = top1_acc message = "The best top1 acc {:.5f}, in epoch: {:d}".format(