From ebadacd7f816770c935bffcc8c3658db8d68c915 Mon Sep 17 00:00:00 2001 From: littletomatodonkey Date: Sat, 17 Oct 2020 17:27:36 +0000 Subject: [PATCH] add acc benchmark --- ppcls/utils/model_zoo.py | 16 +++- tools/benchmark/benchmark.sh | 3 + tools/benchmark/benchmark_acc.py | 114 +++++++++++++++++++++++++++++ tools/benchmark/benchmark_list.txt | 30 ++++++++ tools/download.py | 3 +- tools/eval.py | 4 +- 6 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 tools/benchmark/benchmark.sh create mode 100644 tools/benchmark/benchmark_acc.py create mode 100644 tools/benchmark/benchmark_list.txt diff --git a/ppcls/utils/model_zoo.py b/ppcls/utils/model_zoo.py index d023f4d1..44a11c56 100644 --- a/ppcls/utils/model_zoo.py +++ b/ppcls/utils/model_zoo.py @@ -158,14 +158,22 @@ def _decompress(fname): else: raise TypeError("Unsupport compress file type {}".format(fname)) - for f in os.listdir(fpath_tmp): - src_dir = os.path.join(fpath_tmp, f) - dst_dir = os.path.join(fpath, f) - _move_and_merge_tree(src_dir, dst_dir) + fs = os.listdir(fpath_tmp) + assert len( + fs + ) == 1, "There should just be 1 pretrained path in an archive file but got {}.".format( + len(fs)) + + f = fs[0] + src_dir = os.path.join(fpath_tmp, f) + dst_dir = os.path.join(fpath, f) + _move_and_merge_tree(src_dir, dst_dir) shutil.rmtree(fpath_tmp) os.remove(fname) + return f + def _get_pretrained(): with open('./ppcls/utils/pretrained.list') as flist: diff --git a/tools/benchmark/benchmark.sh b/tools/benchmark/benchmark.sh new file mode 100644 index 00000000..9d44070d --- /dev/null +++ b/tools/benchmark/benchmark.sh @@ -0,0 +1,3 @@ +python3.7 -m paddle.distributed.launch \ + --selected_gpus="5" \ + tools/benchmark/benchmark_acc.py diff --git a/tools/benchmark/benchmark_acc.py b/tools/benchmark/benchmark_acc.py new file mode 100644 index 00000000..61f025e3 --- /dev/null +++ b/tools/benchmark/benchmark_acc.py @@ -0,0 +1,114 @@ +# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import sys +__dir__ = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(__dir__) +sys.path.append(os.path.abspath(os.path.join(__dir__, '..'))) +sys.path.append(os.path.abspath(os.path.join(__dir__, '../..'))) + +from multiprocessing import Process, Manager +import tools.eval as eval +from ppcls.utils.model_zoo import _download, _decompress +from ppcls.utils import logger + + +def parse_args(): + def str2bool(v): + return v.lower() in ("true", "t", "1") + + parser = argparse.ArgumentParser() + parser.add_argument( + "-b", + "--benchmark_file_list", + type=str, + default="./tools/benchmark/benchmark_list.txt") + parser.add_argument( + "-p", "--pretrained_dir", type=str, default="./pretrained/") + + return parser.parse_args() + + +def parse_model_infos(benchmark_file_list): + model_infos = [] + with open(benchmark_file_list, "r") as fin: + lines = fin.readlines() + for idx, line in enumerate(lines): + strs = line.strip("\n").strip("\r").split(" ") + if len(strs) != 4: + logger.info( + "line {0}(info: {1}) format wrong, it should be splited into 4 parts, but got {2}". + format(idx, line, len(strs))) + model_infos.append({ + "top1_acc": float(strs[0]), + "model_name": strs[1], + "config_path": strs[2], + "pretrain_path": strs[3], + }) + return model_infos + + +def main(args): + benchmark_file_list = args.benchmark_file_list + model_infos = parse_model_infos(benchmark_file_list) + right_models = [] + wrong_models = [] + + for model_info in model_infos: + try: + pretrained_url = model_info["pretrain_path"] + fname = _download(pretrained_url, args.pretrained_dir) + pretrained_path = os.path.splitext(fname)[0] + if pretrained_url.endswith("tar"): + path = _decompress(fname) + pretrained_path = os.path.join( + os.path.dirname(pretrained_path), path) + + args.config = model_info["config_path"] + args.override = [ + "pretrained_model={}".format(pretrained_path), + "VALID.batch_size=128", + "VALID.num_workers=16", + "load_static_weights=True", + "print_interval=100", + ] + manager = Manager() + return_dict = manager.dict() + p = Process(target=eval.main, args=(args, return_dict)) + p.start() + p.join() + top1_acc = return_dict.get("top1_acc", 0.0) + except: + top1_acc = 0.0 + diff = abs(top1_acc - model_info["top1_acc"]) + if diff > 0.001: + err_info = "[{}]Top-1 acc diff should be <= 0.001 but got diff {}, gt acc: {}, eval acc: {}".format( + model_info["model_name"], diff, model_info["top1_acc"], + top1_acc) + logger.warning(err_info) + wrong_models.append(model_info["model_name"]) + else: + right_models.append(model_info["model_name"]) + + logger.info("[number of right models: {}, they are: {}".format( + len(right_models), right_models)) + logger.info("[number of wrong models: {}, they are: {}".format( + len(wrong_models), wrong_models)) + + +if __name__ == '__main__': + args = parse_args() + main(args) diff --git a/tools/benchmark/benchmark_list.txt b/tools/benchmark/benchmark_list.txt new file mode 100644 index 00000000..1f8baf13 --- /dev/null +++ b/tools/benchmark/benchmark_list.txt @@ -0,0 +1,30 @@ +0.7098 ResNet18 configs/ResNet/ResNet18.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNet18_pretrained.tar +0.7650 ResNet50 configs/ResNet/ResNet50.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNet50_pretrained.tar +0.7226 ResNet18_vd configs/ResNet/ResNet18_vd.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNet18_vd_pretrained.tar +0.7912 ResNet50_vd configs/ResNet/ResNet50_vd.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNet50_vd_pretrained.tar +0.7099 MobileNetV1 configs/MobileNetV1/MobileNetV1.yaml https://paddle-imagenet-models-name.bj.bcebos.com/MobileNetV1_pretrained.tar +0.7215 MobileNetV2 configs/MobileNetV2/MobileNetV2.yaml https://paddle-imagenet-models-name.bj.bcebos.com/MobileNetV2_pretrained.tar +0.7532 MobileNetV3_large_x1_0 configs/MobileNetV3/MobileNetV3_large_x1_0.yaml https://paddle-imagenet-models-name.bj.bcebos.com/MobileNetV3_large_x1_0_pretrained.tar +0.6880 ShuffleNetV2 configs/ShuffleNet/ShuffleNetV2.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ShuffleNetV2_pretrained.tar +0.7933 Res2Net50_26w_4s configs/Res2Net/Res2Net50_26w_4s.yaml https://paddle-imagenet-models-name.bj.bcebos.com/Res2Net50_26w_4s_pretrained.tar +0.7775 ResNeXt50_32x4d configs/ResNeXt/ResNeXt50_32x4d.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNeXt50_32x4d_pretrained.tar +0.7333 SE_ResNet18_vd configs/SENet/SE_ResNet18_vd.yaml https://paddle-imagenet-models-name.bj.bcebos.com/SE_ResNet18_vd_pretrained.tar +0.7952 SE_ResNet50_vd configs/SENet/SE_ResNet50_vd.yaml https://paddle-imagenet-models-name.bj.bcebos.com/SE_ResNet50_vd_pretrained.tar +0.8024 SE_ResNeXt50_vd_32x4d configs/SENet/SE_ResNeXt50_vd_32x4d.yaml https://paddle-imagenet-models-name.bj.bcebos.com/SE_ResNeXt50_vd_32x4d_pretrained.tar +0.7566 DenseNet121 configs/DenseNet/DenseNet121.yaml https://paddle-imagenet-models-name.bj.bcebos.com/DenseNet121_pretrained.tar +0.7678 DPN68 configs/DPN/DPN68.yaml https://paddle-imagenet-models-name.bj.bcebos.com/DPN68_pretrained.tar +0.7692 HRNet_W18_C configs/HRNet/HRNet_W18_C.yaml https://paddle-imagenet-models-name.bj.bcebos.com/HRNet_W18_C_pretrained.tar +0.7070 GoogLeNet configs/Inception/GoogLeNet.yaml https://paddle-imagenet-models-name.bj.bcebos.com/GoogLeNet_pretrained.tar +0.7930 Xception41 configs/Xception/Xception41.yaml https://paddle-imagenet-models-name.bj.bcebos.com/Xception41_pretrained.tar +0.7955 Xception41_deeplab configs/Xception/Xception41_deeplab.yaml https://paddle-imagenet-models-name.bj.bcebos.com/Xception41_deeplab_pretrained.tar +0.8077 InceptionV4 configs/Inception/InceptionV4.yaml https://paddle-imagenet-models-name.bj.bcebos.com/InceptionV4_pretrained.tar +0.8255 ResNeXt101_32x8d_wsl configs/ResNeXt101_wsl/ResNeXt101_32x8d_wsl.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNeXt101_32x8d_wsl_pretrained.tar +0.7738 EfficientNetB0 configs/EfficientNet/EfficientNetB0.yaml https://paddle-imagenet-models-name.bj.bcebos.com/EfficientNetB0_pretrained.tar +0.8035 ResNeSt50_fast_1s1x64d configs/ResNeSt/ResNeSt50_fast_1s1x64d.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNeSt50_fast_1s1x64d_pretrained.pdparams +0.8102 ResNeSt50 configs/ResNeSt/ResNeSt50.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNeSt50_pretrained.pdparams +0.785 RegNetX_4GF configs/RegNet/RegNetX_4GF.yaml https://paddle-imagenet-models-name.bj.bcebos.com/RegNetX_4GF_pretrained.pdparams +0.7402 GhostNet_x1_0 configs/GhostNet/GhostNet_x1_0.yaml https://paddle-imagenet-models-name.bj.bcebos.com/GhostNet_x1_0_pretrained.pdparams +0.567 AlexNet configs/AlexNet/AlexNet.yaml https://paddle-imagenet-models-name.bj.bcebos.com/AlexNet_pretrained.tar +0.596 SqueezeNet1_0 configs/SqueezeNet/SqueezeNet1_0.yaml https://paddle-imagenet-models-name.bj.bcebos.com/SqueezeNet1_0_pretrained.tar +0.693 VGG11 configs/VGG/VGG11.yaml https://paddle-imagenet-models-name.bj.bcebos.com/VGG11_pretrained.tar +0.780 DarkNet53 configs/DarkNet/DarkNet53.yaml https://paddle-imagenet-models-name.bj.bcebos.com/DarkNet53_ImageNet1k_pretrained.tar diff --git a/tools/download.py b/tools/download.py index a9500efd..1523a781 100644 --- a/tools/download.py +++ b/tools/download.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ppcls import model_zoo import argparse import os import sys @@ -20,6 +19,8 @@ __dir__ = os.path.dirname(os.path.abspath(__file__)) sys.path.append(__dir__) sys.path.append(os.path.abspath(os.path.join(__dir__, '..'))) +from ppcls import model_zoo + def parse_args(): def str2bool(v): diff --git a/tools/eval.py b/tools/eval.py index da37a2c3..f95dcafc 100644 --- a/tools/eval.py +++ b/tools/eval.py @@ -47,7 +47,7 @@ def parse_args(): return args -def main(args): +def main(args, return_dict={}): config = get_config(args.config, overrides=args.override, show=True) # assign place use_gpu = config.get("use_gpu", True) @@ -68,6 +68,8 @@ def main(args): top1_acc = program.run(valid_dataloader, config, net, None, None, 0, 'valid') + return_dict["top1_acc"] = top1_acc + return top1_acc if __name__ == '__main__': -- GitLab