From e3801c551518d6a870c2d914f667395db441d0a6 Mon Sep 17 00:00:00 2001 From: Tingquan Gao Date: Thu, 10 Dec 2020 01:15:24 +0800 Subject: [PATCH] Fix the bug about calling create_predictor repeatedly in hubserving (#462) --- deploy/hubserving/clas/module.py | 5 +-- deploy/hubserving/readme.md | 2 +- deploy/hubserving/readme_en.md | 2 +- tools/infer/predict.py | 61 +++++++++----------------------- tools/infer/utils.py | 30 ++++++++++++++++ 5 files changed, 51 insertions(+), 49 deletions(-) diff --git a/deploy/hubserving/clas/module.py b/deploy/hubserving/clas/module.py index 49f3b9de..7abc1c3f 100644 --- a/deploy/hubserving/clas/module.py +++ b/deploy/hubserving/clas/module.py @@ -25,7 +25,7 @@ import numpy as np import paddlehub as hub import tools.infer.predict as paddle_predict -from tools.infer.utils import Base64ToCV2 +from tools.infer.utils import Base64ToCV2, create_paddle_predictor from deploy.hubserving.clas.params import read_params @@ -96,6 +96,7 @@ class ClasSystem(hub.Module): assert predicted_data != [], "There is not any image to be predicted. Please check the input data." + predictor = create_paddle_predictor(self.args) all_results = [] for img in predicted_data: if img is None: @@ -106,7 +107,7 @@ class ClasSystem(hub.Module): self.args.image_file = img self.args.top_k = top_k - classes, scores = paddle_predict.main(self.args) + classes, scores = paddle_predict.predict(self.args, predictor) elapse = time.time() - starttime logger.info("Predict time: {}".format(elapse)) diff --git a/deploy/hubserving/readme.md b/deploy/hubserving/readme.md index ab97083d..65fb9e7b 100644 --- a/deploy/hubserving/readme.md +++ b/deploy/hubserving/readme.md @@ -138,7 +138,7 @@ hub serving start -c deploy/hubserving/clas/config.json ```hub uninstall clas_system``` - 4、 安装修改后的新服务包 -```hub install deploy/hubserving/clas_system/``` +```hub install deploy/hubserving/clas/``` - 5、重新启动服务 ```hub serving start -m clas_system``` diff --git a/deploy/hubserving/readme_en.md b/deploy/hubserving/readme_en.md index ded2e993..1cb93ba4 100644 --- a/deploy/hubserving/readme_en.md +++ b/deploy/hubserving/readme_en.md @@ -144,7 +144,7 @@ hub uninstall clas_system ``` - 4. Install modified service module ```shell -hub install deploy/hubserving/clas_system/ +hub install deploy/hubserving/clas/ ``` - 5. Restart service ```shell diff --git a/tools/infer/predict.py b/tools/infer/predict.py index ec968468..885bb337 100644 --- a/tools/infer/predict.py +++ b/tools/infer/predict.py @@ -19,51 +19,8 @@ import numpy as np import cv2 import time -from paddle.inference import Config -from paddle.inference import create_predictor - - -def create_paddle_predictor(args): - config = Config(args.model_file, args.params_file) - - if args.use_gpu: - config.enable_use_gpu(args.gpu_mem, 0) - else: - config.disable_gpu() - if args.enable_mkldnn: - # cache 10 different shapes for mkldnn to avoid memory leak - config.set_mkldnn_cache_capacity(10) - config.enable_mkldnn() - - config.disable_glog_info() - config.switch_ir_optim(args.ir_optim) # default true - if args.use_tensorrt: - config.enable_tensorrt_engine( - precision_mode=Config.Precision.Half - if args.use_fp16 else Config.Precision.Float32, - max_batch_size=args.batch_size) - - config.enable_memory_optim() - # use zero copy - config.switch_use_feed_fetch_ops(False) - predictor = create_predictor(config) - - return predictor - - -def main(args): - if not args.enable_benchmark: - assert args.batch_size == 1 - assert args.use_fp16 is False - else: - assert args.use_gpu is True - assert args.model is not None - # HALF precission predict only work when using tensorrt - if args.use_fp16 is True: - assert args.use_tensorrt is True - - predictor = create_paddle_predictor(args) +def predict(args, predictor): input_names = predictor.get_input_names() input_tensor = predictor.get_input_handle(input_names[0]) @@ -91,10 +48,11 @@ def main(args): output = output_tensor.copy_to_cpu() classes, scores = utils.postprocess(output, args) + if args.hubserving: + return classes, scores print("Current image file: {}".format(args.image_file)) print("\ttop-1 class: {0}".format(classes[0])) print("\ttop-1 score: {0}".format(scores[0])) - else: for i in range(0, test_num + 10): inputs = np.random.rand(args.batch_size, 3, 224, @@ -117,6 +75,19 @@ def main(args): / test_num)) +def main(args): + if not args.enable_benchmark: + assert args.batch_size == 1 + else: + assert args.model is not None + # HALF precission predict only work when using tensorrt + if args.use_fp16 is True: + assert args.use_tensorrt is True + + predictor = utils.create_paddle_predictor(args) + predict(args, predictor) + + if __name__ == "__main__": args = utils.parse_args() main(args) diff --git a/tools/infer/utils.py b/tools/infer/utils.py index 67e37926..f31fca91 100644 --- a/tools/infer/utils.py +++ b/tools/infer/utils.py @@ -15,6 +15,8 @@ import argparse import cv2 import numpy as np +from paddle.inference import Config +from paddle.inference import create_predictor def parse_args(): @@ -65,6 +67,34 @@ def parse_args(): return parser.parse_args() +def create_paddle_predictor(args): + config = Config(args.model_file, args.params_file) + + if args.use_gpu: + config.enable_use_gpu(args.gpu_mem, 0) + else: + config.disable_gpu() + if args.enable_mkldnn: + # cache 10 different shapes for mkldnn to avoid memory leak + config.set_mkldnn_cache_capacity(10) + config.enable_mkldnn() + + config.disable_glog_info() + config.switch_ir_optim(args.ir_optim) # default true + if args.use_tensorrt: + config.enable_tensorrt_engine( + precision_mode=Config.Precision.Half + if args.use_fp16 else Config.Precision.Float32, + max_batch_size=args.batch_size) + + config.enable_memory_optim() + # use zero copy + config.switch_use_feed_fetch_ops(False) + predictor = create_predictor(config) + + return predictor + + def preprocess(img, args): resize_op = ResizeImage(resize_short=args.resize_short) img = resize_op(img) -- GitLab