Merge pull request #312 from littletomatodonkey/dyg/add_acc_bench

add acc benchmark

ppcls/utils/model_zoo.py

@@ -158,7 +158,13 @@ def _decompress(fname):
     else:
         raise TypeError("Unsupport compress file type {}".format(fname))
 
-    for f in os.listdir(fpath_tmp):
+    fs = os.listdir(fpath_tmp)
+    assert len(
+        fs
+    ) == 1, "There should just be 1 pretrained path in an archive file but got {}.".format(
+        len(fs))
+
+    f = fs[0]
     src_dir = os.path.join(fpath_tmp, f)
     dst_dir = os.path.join(fpath, f)
     _move_and_merge_tree(src_dir, dst_dir)
@@ -166,6 +172,8 @@ def _decompress(fname):
     shutil.rmtree(fpath_tmp)
     os.remove(fname)
 
+    return f
+
 
 def _get_pretrained():
     with open('./ppcls/utils/pretrained.list') as flist:

tools/benchmark/benchmark.sh

+python3.7 -m paddle.distributed.launch  \
+	--selected_gpus="0" \
+	tools/benchmark/benchmark_acc.py

tools/benchmark/benchmark_acc.py

+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import sys
+__dir__ = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(__dir__)
+sys.path.append(os.path.abspath(os.path.join(__dir__, '..')))
+sys.path.append(os.path.abspath(os.path.join(__dir__, '../..')))
+
+from multiprocessing import Process, Manager
+import tools.eval as eval
+from ppcls.utils.model_zoo import _download, _decompress
+from ppcls.utils import logger
+
+
+def parse_args():
+    def str2bool(v):
+        return v.lower() in ("true", "t", "1")
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-b",
+        "--benchmark_file_list",
+        type=str,
+        default="./tools/benchmark/benchmark_list.txt")
+    parser.add_argument(
+        "-p", "--pretrained_dir", type=str, default="./pretrained/")
+
+    return parser.parse_args()
+
+
+def parse_model_infos(benchmark_file_list):
+    model_infos = []
+    with open(benchmark_file_list, "r") as fin:
+        lines = fin.readlines()
+        for idx, line in enumerate(lines):
+            strs = line.strip("\n").strip("\r").split(" ")
+            if len(strs) != 4:
+                logger.info(
+                    "line {0}(info: {1}) format wrong, it should be splited into 4 parts, but got {2}".
+                    format(idx, line, len(strs)))
+            model_infos.append({
+                "top1_acc": float(strs[0]),
+                "model_name": strs[1],
+                "config_path": strs[2],
+                "pretrain_path": strs[3],
+            })
+    return model_infos
+
+
+def main(args):
+    benchmark_file_list = args.benchmark_file_list
+    model_infos = parse_model_infos(benchmark_file_list)
+    right_models = []
+    wrong_models = []
+
+    for model_info in model_infos:
+        try:
+            pretrained_url = model_info["pretrain_path"]
+            fname = _download(pretrained_url, args.pretrained_dir)
+            pretrained_path = os.path.splitext(fname)[0]
+            if pretrained_url.endswith("tar"):
+                path = _decompress(fname)
+                pretrained_path = os.path.join(
+                    os.path.dirname(pretrained_path), path)
+
+            args.config = model_info["config_path"]
+            args.override = [
+                "pretrained_model={}".format(pretrained_path),
+                "VALID.batch_size=128",
+                "VALID.num_workers=16",
+                "load_static_weights=True",
+                "print_interval=100",
+            ]
+            manager = Manager()
+            return_dict = manager.dict()
+            p = Process(target=eval.main, args=(args, return_dict))
+            p.start()
+            p.join()
+            top1_acc = return_dict.get("top1_acc", 0.0)
+        except:
+            top1_acc = 0.0
+        diff = abs(top1_acc - model_info["top1_acc"])
+        if diff > 0.001:
+            err_info = "[{}]Top-1 acc diff should be <= 0.001 but got diff {}, gt acc: {}, eval acc: {}".format(
+                model_info["model_name"], diff, model_info["top1_acc"],
+                top1_acc)
+            logger.warning(err_info)
+            wrong_models.append(model_info["model_name"])
+        else:
+            right_models.append(model_info["model_name"])
+
+    logger.info("[number of right models: {}, they are: {}".format(
+        len(right_models), right_models))
+    logger.info("[number of wrong models: {}, they are: {}".format(
+        len(wrong_models), wrong_models))
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    main(args)

tools/benchmark/benchmark_list.txt

+0.7098 ResNet18 configs/ResNet/ResNet18.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNet18_pretrained.tar
+0.7650 ResNet50 configs/ResNet/ResNet50.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNet50_pretrained.tar
+0.7226 ResNet18_vd configs/ResNet/ResNet18_vd.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNet18_vd_pretrained.tar
+0.7912 ResNet50_vd configs/ResNet/ResNet50_vd.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNet50_vd_pretrained.tar
+0.7099 MobileNetV1 configs/MobileNetV1/MobileNetV1.yaml https://paddle-imagenet-models-name.bj.bcebos.com/MobileNetV1_pretrained.tar
+0.7215 MobileNetV2 configs/MobileNetV2/MobileNetV2.yaml https://paddle-imagenet-models-name.bj.bcebos.com/MobileNetV2_pretrained.tar
+0.7532 MobileNetV3_large_x1_0 configs/MobileNetV3/MobileNetV3_large_x1_0.yaml https://paddle-imagenet-models-name.bj.bcebos.com/MobileNetV3_large_x1_0_pretrained.tar
+0.6880 ShuffleNetV2 configs/ShuffleNet/ShuffleNetV2.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ShuffleNetV2_pretrained.tar
+0.7933 Res2Net50_26w_4s configs/Res2Net/Res2Net50_26w_4s.yaml https://paddle-imagenet-models-name.bj.bcebos.com/Res2Net50_26w_4s_pretrained.tar
+0.7775 ResNeXt50_32x4d configs/ResNeXt/ResNeXt50_32x4d.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNeXt50_32x4d_pretrained.tar
+0.7333 SE_ResNet18_vd configs/SENet/SE_ResNet18_vd.yaml https://paddle-imagenet-models-name.bj.bcebos.com/SE_ResNet18_vd_pretrained.tar
+0.7952 SE_ResNet50_vd configs/SENet/SE_ResNet50_vd.yaml https://paddle-imagenet-models-name.bj.bcebos.com/SE_ResNet50_vd_pretrained.tar
+0.8024 SE_ResNeXt50_vd_32x4d configs/SENet/SE_ResNeXt50_vd_32x4d.yaml https://paddle-imagenet-models-name.bj.bcebos.com/SE_ResNeXt50_vd_32x4d_pretrained.tar
+0.7566 DenseNet121 configs/DenseNet/DenseNet121.yaml https://paddle-imagenet-models-name.bj.bcebos.com/DenseNet121_pretrained.tar
+0.7678 DPN68 configs/DPN/DPN68.yaml https://paddle-imagenet-models-name.bj.bcebos.com/DPN68_pretrained.tar
+0.7692 HRNet_W18_C configs/HRNet/HRNet_W18_C.yaml https://paddle-imagenet-models-name.bj.bcebos.com/HRNet_W18_C_pretrained.tar
+0.7070 GoogLeNet configs/Inception/GoogLeNet.yaml https://paddle-imagenet-models-name.bj.bcebos.com/GoogLeNet_pretrained.tar
+0.7930 Xception41 configs/Xception/Xception41.yaml https://paddle-imagenet-models-name.bj.bcebos.com/Xception41_pretrained.tar
+0.7955 Xception41_deeplab configs/Xception/Xception41_deeplab.yaml https://paddle-imagenet-models-name.bj.bcebos.com/Xception41_deeplab_pretrained.tar
+0.8077 InceptionV4 configs/Inception/InceptionV4.yaml https://paddle-imagenet-models-name.bj.bcebos.com/InceptionV4_pretrained.tar
+0.8255 ResNeXt101_32x8d_wsl configs/ResNeXt101_wsl/ResNeXt101_32x8d_wsl.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNeXt101_32x8d_wsl_pretrained.tar
+0.8035 ResNeSt50_fast_1s1x64d configs/ResNeSt/ResNeSt50_fast_1s1x64d.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNeSt50_fast_1s1x64d_pretrained.pdparams
+0.8102 ResNeSt50 configs/ResNeSt/ResNeSt50.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNeSt50_pretrained.pdparams
+0.785 RegNetX_4GF configs/RegNet/RegNetX_4GF.yaml https://paddle-imagenet-models-name.bj.bcebos.com/RegNetX_4GF_pretrained.pdparams
+0.7402 GhostNet_x1_0 configs/GhostNet/GhostNet_x1_0.yaml https://paddle-imagenet-models-name.bj.bcebos.com/GhostNet_x1_0_pretrained.pdparams
+0.567 AlexNet configs/AlexNet/AlexNet.yaml https://paddle-imagenet-models-name.bj.bcebos.com/AlexNet_pretrained.tar
+0.596 SqueezeNet1_0 configs/SqueezeNet/SqueezeNet1_0.yaml https://paddle-imagenet-models-name.bj.bcebos.com/SqueezeNet1_0_pretrained.tar
+0.693 VGG11 configs/VGG/VGG11.yaml https://paddle-imagenet-models-name.bj.bcebos.com/VGG11_pretrained.tar
+0.780 DarkNet53 configs/DarkNet/DarkNet53.yaml https://paddle-imagenet-models-name.bj.bcebos.com/DarkNet53_ImageNet1k_pretrained.tar

tools/download.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ppcls import model_zoo
 import argparse
 import os
 import sys
@@ -20,6 +19,8 @@ __dir__ = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(__dir__)
 sys.path.append(os.path.abspath(os.path.join(__dir__, '..')))
 
+from ppcls import model_zoo
+
 
 def parse_args():
     def str2bool(v):

tools/eval.py

@@ -47,7 +47,7 @@ def parse_args():
     return args
 
 
-def main(args):
+def main(args, return_dict={}):
     config = get_config(args.config, overrides=args.override, show=True)
     # assign place
     use_gpu = config.get("use_gpu", True)
@@ -68,6 +68,8 @@ def main(args):
 
     top1_acc = program.run(valid_dataloader, config, net, None, None, 0,
                            'valid')
+    return_dict["top1_acc"] = top1_acc
+    return top1_acc
 
 
 if __name__ == '__main__':