Add check_config and logger num_classes (#593)

* update check config

* fix slim/xx

ppdet/utils/check.py

@@ -23,7 +23,7 @@ import paddle.fluid as fluid
 import logging
 logger = logging.getLogger(__name__)
 
-__all__ = ['check_gpu', 'check_version']
+__all__ = ['check_gpu', 'check_version', 'check_config']
 
 
 def check_gpu(use_gpu):
@@ -59,3 +59,40 @@ def check_version():
     except Exception as e:
         logger.error(err)
         sys.exit(1)
+
+
+def check_config(cfg):
+    """
+    Check the correctness of the configuration file. Log error and exit
+    when Config is not compliant.
+    """
+    err = "'{}' not specified in config file. Please set it in config file."
+    check_list = ['architecture', 'num_classes']
+    try:
+        for var in check_list:
+            if not var in cfg:
+                logger.error(err.format(var))
+                sys.exit(1)
+    except Exception as e:
+        pass
+
+    if 'log_iter' not in cfg:
+        cfg.log_iter = 20
+
+    train_dataset = cfg['TrainReader']['dataset']
+    eval_dataset = cfg['EvalReader']['dataset']
+    test_dataset = cfg['TestReader']['dataset']
+    assert train_dataset.with_background == eval_dataset.with_background, \
+        "'with_background' of TrainReader is not equal to EvalReader."
+    assert train_dataset.with_background == test_dataset.with_background, \
+        "'with_background' of TrainReader is not equal to TestReader."
+
+    actual_num_classes = int(cfg.num_classes) - int(
+        train_dataset.with_background)
+    logger.info("The 'num_classes'(number of classes) you set is {}, " \
+                "and 'with_background' in 'dataset' sets {}.\n" \
+                "So please note the actual number of categories is {}."
+                .format(cfg.num_classes, train_dataset.with_background,
+                    actual_num_classes))
+
+    return cfg

slim/distillation/distill.py

@@ -27,7 +27,7 @@ from ppdet.data.reader import create_reader
 from ppdet.utils.eval_utils import parse_fetches, eval_results, eval_run
 from ppdet.utils.stats import TrainingStats
 from ppdet.utils.cli import ArgsParser
-from ppdet.utils.check import check_gpu
+from ppdet.utils.check import check_gpu, check_config
 import ppdet.utils.checkpoint as checkpoint
 
 import logging
@@ -125,18 +125,13 @@ def split_distill(split_output_names, weight):
 def main():
     env = os.environ
     cfg = load_config(FLAGS.config)
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
-    if 'log_iter' not in cfg:
-        cfg.log_iter = 20
-
+    check_config(cfg)
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
 
+    main_arch = cfg.architecture
+
     if cfg.use_gpu:
         devices_num = fluid.core.get_cuda_device_count()
     else:

slim/extensions/distill_pruned_model/distill_pruned_model.py

@@ -29,7 +29,7 @@ from ppdet.data.reader import create_reader
 from ppdet.utils.eval_utils import parse_fetches, eval_results, eval_run
 from ppdet.utils.stats import TrainingStats
 from ppdet.utils.cli import ArgsParser
-from ppdet.utils.check import check_gpu
+from ppdet.utils.check import check_gpu, check_config
 import ppdet.utils.checkpoint as checkpoint
 
 import logging
@@ -113,18 +113,13 @@ def split_distill(split_output_names, weight):
 def main():
     env = os.environ
     cfg = load_config(FLAGS.config)
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
-    if 'log_iter' not in cfg:
-        cfg.log_iter = 20
-
+    check_config(cfg)
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
 
+    main_arch = cfg.architecture
+
     if cfg.use_gpu:
         devices_num = fluid.core.get_cuda_device_count()
     else:

slim/nas/train_nas.py

@@ -47,7 +47,7 @@ from ppdet.utils import dist_utils
 from ppdet.utils.eval_utils import parse_fetches, eval_run
 from ppdet.utils.stats import TrainingStats
 from ppdet.utils.cli import ArgsParser
-from ppdet.utils.check import check_gpu, check_version
+from ppdet.utils.check import check_gpu, check_version, check_config
 import ppdet.utils.checkpoint as checkpoint
 from paddleslim.analysis import flops, TableLatencyEvaluator
 from paddleslim.nas import SANAS
@@ -209,21 +209,15 @@ def main():
         np.random.seed(local_seed)
 
     cfg = load_config(FLAGS.config)
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
-
-    if 'log_iter' not in cfg:
-        cfg.log_iter = 20
-
+    check_config(cfg)
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
     # check if paddlepaddle version is satisfied
     check_version()
 
+    main_arch = cfg.architecture
+
     if cfg.use_gpu:
         devices_num = fluid.core.get_cuda_device_count()
     else:

slim/prune/eval.py

@@ -37,7 +37,7 @@ from paddleslim.analysis import flops
 
 from ppdet.utils.eval_utils import parse_fetches, eval_run, eval_results, json_eval_results
 import ppdet.utils.checkpoint as checkpoint
-from ppdet.utils.check import check_gpu, check_version
+from ppdet.utils.check import check_gpu, check_version, check_config
 
 from ppdet.data.reader import create_reader
 
@@ -55,17 +55,15 @@ def main():
     Main evaluate function
     """
     cfg = load_config(FLAGS.config)
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
+    check_config(cfg)
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
     # check if paddlepaddle version is satisfied
     check_version()
 
+    main_arch = cfg.architecture
+
     multi_scale_test = getattr(cfg, 'MultiScaleTEST', None)
 
     # define executor

slim/prune/export_model.py

@@ -23,6 +23,7 @@ from paddle import fluid
 from ppdet.core.workspace import load_config, merge_config, create
 from ppdet.utils.cli import ArgsParser
 import ppdet.utils.checkpoint as checkpoint
+from ppdet.utils.check import check_config
 from paddleslim.prune import Pruner
 from paddleslim.analysis import flops
 
@@ -75,13 +76,10 @@ def save_infer_model(FLAGS, exe, feed_vars, test_fetches, infer_prog):
 
 def main():
     cfg = load_config(FLAGS.config)
-
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
+    check_config(cfg)
+
+    main_arch = cfg.architecture
 
     # Use CPU for exporting inference model instead of GPU
     place = fluid.CPUPlace()

slim/prune/prune.py

@@ -27,12 +27,11 @@ from paddle import fluid
 from ppdet.experimental import mixed_precision_context
 from ppdet.core.workspace import load_config, merge_config, create
 from ppdet.data.reader import create_reader
-from ppdet.utils.cli import print_total_cfg
 from ppdet.utils import dist_utils
 from ppdet.utils.eval_utils import parse_fetches, eval_run, eval_results
 from ppdet.utils.stats import TrainingStats
 from ppdet.utils.cli import ArgsParser
-from ppdet.utils.check import check_gpu, check_version
+from ppdet.utils.check import check_gpu, check_version, check_config
 import ppdet.utils.checkpoint as checkpoint
 
 import logging
@@ -52,22 +51,14 @@ def main():
         np.random.seed(local_seed)
 
     cfg = load_config(FLAGS.config)
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
-
-    if 'log_iter' not in cfg:
-        cfg.log_iter = 20
-
+    check_config(cfg)
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
     # check if paddlepaddle version is satisfied
     check_version()
-    if not FLAGS.dist or trainer_id == 0:
-        print_total_cfg(cfg)
+
+    main_arch = cfg.architecture
 
     if cfg.use_gpu:
         devices_num = fluid.core.get_cuda_device_count()

slim/quantization/eval.py

@@ -23,7 +23,7 @@ import paddle.fluid as fluid
 
 from ppdet.utils.eval_utils import parse_fetches, eval_run, eval_results, json_eval_results
 import ppdet.utils.checkpoint as checkpoint
-from ppdet.utils.check import check_gpu, check_version
+from ppdet.utils.check import check_gpu, check_version, check_config
 
 from ppdet.data.reader import create_reader
 
@@ -44,17 +44,15 @@ def main():
     Main evaluate function
     """
     cfg = load_config(FLAGS.config)
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
+    check_config(cfg)
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
     # check if paddlepaddle version is satisfied
     check_version()
 
+    main_arch = cfg.architecture
+
     # define executor
     place = fluid.CUDAPlace(0) if cfg.use_gpu else fluid.CPUPlace()
     exe = fluid.Executor(place)

slim/quantization/export_model.py

@@ -23,6 +23,7 @@ from paddle import fluid
 from ppdet.core.workspace import load_config, merge_config, create
 from ppdet.utils.cli import ArgsParser
 import ppdet.utils.checkpoint as checkpoint
+from ppdet.utils.check import check_config
 from tools.export_model import prune_feed_vars
 
 import logging
@@ -50,13 +51,10 @@ def save_infer_model(save_dir, exe, feed_vars, test_fetches, infer_prog):
 
 def main():
     cfg = load_config(FLAGS.config)
-
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
+    check_config(cfg)
+
+    main_arch = cfg.architecture
 
     # Use CPU for exporting inference model instead of GPU
     place = fluid.CPUPlace()

slim/quantization/infer.py

@@ -29,7 +29,7 @@ from ppdet.core.workspace import load_config, merge_config, create
 
 from ppdet.utils.eval_utils import parse_fetches
 from ppdet.utils.cli import ArgsParser
-from ppdet.utils.check import check_gpu, check_version
+from ppdet.utils.check import check_gpu, check_version, check_config
 from ppdet.utils.visualizer import visualize_results
 import ppdet.utils.checkpoint as checkpoint
 
@@ -44,19 +44,15 @@ from paddleslim.quant import quant_aware, convert
 
 def main():
     cfg = load_config(FLAGS.config)
-
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
-
+    check_config(cfg)
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
     # check if paddlepaddle version is satisfied
     check_version()
 
+    main_arch = cfg.architecture
+
     dataset = cfg.TestReader['dataset']
 
     test_images = get_test_images(FLAGS.infer_dir, FLAGS.infer_img)

slim/quantization/train.py

@@ -29,12 +29,11 @@ from paddle import fluid
 from ppdet.core.workspace import load_config, merge_config, create
 from ppdet.data.reader import create_reader
 
-from ppdet.utils.cli import print_total_cfg
 from ppdet.utils import dist_utils
 from ppdet.utils.eval_utils import parse_fetches, eval_run, eval_results
 from ppdet.utils.stats import TrainingStats
 from ppdet.utils.cli import ArgsParser
-from ppdet.utils.check import check_gpu, check_version
+from ppdet.utils.check import check_gpu, check_version, check_config
 import ppdet.utils.checkpoint as checkpoint
 from paddleslim.quant import quant_aware, convert
 import logging
@@ -73,22 +72,14 @@ def main():
         np.random.seed(local_seed)
 
     cfg = load_config(FLAGS.config)
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
-
-    if 'log_iter' not in cfg:
-        cfg.log_iter = 20
-
+    check_config(cfg)
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
     # check if paddlepaddle version is satisfied
     check_version()
-    if not FLAGS.dist or trainer_id == 0:
-        print_total_cfg(cfg)
+
+    main_arch = cfg.architecture
 
     if cfg.use_gpu:
         devices_num = fluid.core.get_cuda_device_count()

slim/sensitive/sensitive.py

@@ -41,12 +41,11 @@ from ppdet.core.workspace import load_config, merge_config, create
 
 from ppdet.data.reader import create_reader
 
-from ppdet.utils.cli import print_total_cfg
 from ppdet.utils import dist_utils
 from ppdet.utils.eval_utils import parse_fetches, eval_run, eval_results
 from ppdet.utils.stats import TrainingStats
 from ppdet.utils.cli import ArgsParser
-from ppdet.utils.check import check_gpu, check_version
+from ppdet.utils.check import check_gpu, check_version, check_config
 import ppdet.utils.checkpoint as checkpoint
 from paddleslim.prune import sensitivity
 import logging
@@ -60,12 +59,10 @@ def main():
 
     print("FLAGS.config: {}".format(FLAGS.config))
     cfg = load_config(FLAGS.config)
-    assert 'architecture' in cfg
-    main_arch = cfg.architecture
-
     merge_config(FLAGS.opt)
+    check_config(cfg)
 
-    print_total_cfg(cfg)
+    main_arch = cfg.architecture
 
     place = fluid.CUDAPlace(0)
     exe = fluid.Executor(place)

tools/eval.py

@@ -35,7 +35,7 @@ import paddle.fluid as fluid
 
 from ppdet.utils.eval_utils import parse_fetches, eval_run, eval_results, json_eval_results
 import ppdet.utils.checkpoint as checkpoint
-from ppdet.utils.check import check_gpu, check_version
+from ppdet.utils.check import check_gpu, check_version, check_config
 
 from ppdet.data.reader import create_reader
 
@@ -53,17 +53,15 @@ def main():
     Main evaluate function
     """
     cfg = load_config(FLAGS.config)
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
+    check_config(cfg)
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
     # check if paddlepaddle version is satisfied
     check_version()
 
+    main_arch = cfg.architecture
+
     multi_scale_test = getattr(cfg, 'MultiScaleTEST', None)
 
     # define executor

tools/export_model.py

@@ -23,6 +23,7 @@ from paddle import fluid
 from ppdet.core.workspace import load_config, merge_config, create
 from ppdet.utils.cli import ArgsParser
 import ppdet.utils.checkpoint as checkpoint
+from ppdet.utils.check import check_config
 import yaml
 import logging
 from collections import OrderedDict
@@ -166,13 +167,10 @@ def save_infer_model(FLAGS, exe, feed_vars, test_fetches, infer_prog):
 
 def main():
     cfg = load_config(FLAGS.config)
-
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
+    check_config(cfg)
+
+    main_arch = cfg.architecture
 
     # Use CPU for exporting inference model instead of GPU
     place = fluid.CPUPlace()

tools/face_eval.py

@@ -210,16 +210,13 @@ def main():
     Main evaluate function
     """
     cfg = load_config(FLAGS.config)
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
-
+    check_config(cfg)
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
 
+    main_arch = cfg.architecture
+
     # define executor
     place = fluid.CUDAPlace(0) if cfg.use_gpu else fluid.CPUPlace()
     exe = fluid.Executor(place)

tools/infer.py

@@ -41,7 +41,7 @@ from ppdet.core.workspace import load_config, merge_config, create
 
 from ppdet.utils.eval_utils import parse_fetches
 from ppdet.utils.cli import ArgsParser
-from ppdet.utils.check import check_gpu, check_version
+from ppdet.utils.check import check_gpu, check_version, check_config
 from ppdet.utils.visualizer import visualize_results
 import ppdet.utils.checkpoint as checkpoint
 
@@ -98,18 +98,15 @@ def get_test_images(infer_dir, infer_img):
 def main():
     cfg = load_config(FLAGS.config)
 
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
-
+    check_config(cfg)
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
     # check if paddlepaddle version is satisfied
     check_version()
 
+    main_arch = cfg.architecture
+
     dataset = cfg.TestReader['dataset']
 
     test_images = get_test_images(FLAGS.infer_dir, FLAGS.infer_img)

tools/train.py

@@ -49,7 +49,7 @@ from ppdet.utils import dist_utils
 from ppdet.utils.eval_utils import parse_fetches, eval_run, eval_results
 from ppdet.utils.stats import TrainingStats
 from ppdet.utils.cli import ArgsParser
-from ppdet.utils.check import check_gpu, check_version
+from ppdet.utils.check import check_gpu, check_version, check_config
 import ppdet.utils.checkpoint as checkpoint
 
 import logging
@@ -72,21 +72,15 @@ def main():
         np.random.seed(0)
 
     cfg = load_config(FLAGS.config)
-    if 'architecture' in cfg:
-        main_arch = cfg.architecture
-    else:
-        raise ValueError("'architecture' not specified in config file.")
-
     merge_config(FLAGS.opt)
-
-    if 'log_iter' not in cfg:
-        cfg.log_iter = 20
-
+    check_config(cfg)
     # check if set use_gpu=True in paddlepaddle cpu version
     check_gpu(cfg.use_gpu)
     # check if paddlepaddle version is satisfied
     check_version()
 
+    main_arch = cfg.architecture
+
     if cfg.use_gpu:
         devices_num = fluid.core.get_cuda_device_count()
     else: