diff --git a/deploy/paddle2onnx/readme.md b/deploy/paddle2onnx/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..5db45ad9ce8bcd19377229f5ab44cea74027d16a --- /dev/null +++ b/deploy/paddle2onnx/readme.md @@ -0,0 +1,61 @@ +# paddle2onnx 模型转化与预测 + +本章节介绍 ResNet50_vd 模型如何转化为 ONNX 模型，并基于 ONNX 引擎预测。 + +## 1. 环境准备 + +需要准备 Paddle2ONNX 模型转化环境，和 ONNX 模型预测环境。 + +Paddle2ONNX 支持将 PaddlePaddle 模型格式转化到 ONNX 模型格式，算子目前稳定支持导出 ONNX Opset 9~11，部分Paddle算子支持更低的ONNX Opset转换。 +更多细节可参考 [Paddle2ONNX](https://github.com/PaddlePaddle/Paddle2ONNX/blob/develop/README_zh.md) + +- 安装 Paddle2ONNX +``` +python3.7 -m pip install paddle2onnx +``` + +- 安装 ONNX 运行时 +``` +python3.7 -m pip install onnxruntime +``` + +## 2. 模型转换 + +- ResNet50_vd inference模型下载 + +``` +cd deploy +mkdir models && cd models +wget -nc https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/inference/ResNet50_vd_infer.tar && tar xf ResNet50_vd_infer.tar +cd .. +``` + +- 模型转换 + +使用 Paddle2ONNX 将 Paddle 静态图模型转换为 ONNX 模型格式： +``` +paddle2onnx --model_dir=./models/ResNet50_vd_infer/ \ +--model_filename=inference.pdmodel \ +--params_filename=inference.pdiparams \ +--save_file=./models/ResNet50_vd_infer/inference.onnx \ +--opset_version=10 \ +--enable_onnx_checker=True +``` + +执行完毕后，ONNX 模型 `inference.onnx` 会被保存在 `./models/ResNet50_vd_infer/` 路径下 + +## 3. onnx 预测 + +执行如下命令： +``` +python3.7 python/predict_cls.py \ +-c configs/inference_cls.yaml \ +-o Global.use_onnx=True \ +-o Global.use_gpu=False \ +-o Global.inference_model_dir=./models/ResNet50_vd_infer \ +``` + +结果如下： +``` +ILSVRC2012_val_00000010.jpeg: class id(s): [153, 204, 229, 332, 155], score(s): [0.69, 0.10, 0.02, 0.01, 0.01], label_name(s): ['Maltese dog, Maltese terrier, Maltese', 'Lhasa, Lhasa apso', 'Old English sheepdog, bobtail', 'Angora, Angora rabbit', 'Shih-Tzu'] +``` diff --git a/deploy/python/predict_cls.py b/deploy/python/predict_cls.py index 7d6d861648e1629a89fd255027a9e4798d09547d..574caa3e73bffee4fbf86224f5d91bc7965694b1 100644 --- a/deploy/python/predict_cls.py +++ b/deploy/python/predict_cls.py @@ -67,12 +67,17 @@ class ClsPredictor(Predictor): warmup=2) def predict(self, images): - input_names = self.paddle_predictor.get_input_names() - input_tensor = self.paddle_predictor.get_input_handle(input_names[0]) + use_onnx = self.args.get("use_onnx", False) + if not use_onnx: + input_names = self.predictor.get_input_names() + input_tensor = self.predictor.get_input_handle(input_names[0]) + + output_names = self.predictor.get_output_names() + output_tensor = self.predictor.get_output_handle(output_names[0]) + else: + input_names = self.predictor.get_inputs()[0].name + output_names = self.predictor.get_outputs()[0].name - output_names = self.paddle_predictor.get_output_names() - output_tensor = self.paddle_predictor.get_output_handle(output_names[ - 0]) if self.benchmark: self.auto_logger.times.start() if not isinstance(images, (list, )): @@ -84,9 +89,15 @@ class ClsPredictor(Predictor): if self.benchmark: self.auto_logger.times.stamp() - input_tensor.copy_from_cpu(image) - self.paddle_predictor.run() - batch_output = output_tensor.copy_to_cpu() + if not use_onnx: + input_tensor.copy_from_cpu(image) + self.predictor.run() + batch_output = output_tensor.copy_to_cpu() + else: + batch_output = self.predictor.run( + output_names=[output_names], + input_feed={input_names: image})[0] + if self.benchmark: self.auto_logger.times.stamp() if self.postprocess is not None: diff --git a/deploy/python/predict_det.py b/deploy/python/predict_det.py index 7b23e6220ffc680815ffc6b3eb59f493ddfe9e3c..e4e0a24a6dbc6c62f82810c865096f768ebd182b 100644 --- a/deploy/python/predict_det.py +++ b/deploy/python/predict_det.py @@ -109,17 +109,16 @@ class DetPredictor(Predictor): ''' inputs = self.preprocess(image) np_boxes = None - input_names = self.paddle_predictor.get_input_names() + input_names = self.predictor.get_input_names() for i in range(len(input_names)): - input_tensor = self.paddle_predictor.get_input_handle(input_names[ - i]) + input_tensor = self.predictor.get_input_handle(input_names[i]) input_tensor.copy_from_cpu(inputs[input_names[i]]) t1 = time.time() - self.paddle_predictor.run() - output_names = self.paddle_predictor.get_output_names() - boxes_tensor = self.paddle_predictor.get_output_handle(output_names[0]) + self.predictor.run() + output_names = self.predictor.get_output_names() + boxes_tensor = self.predictor.get_output_handle(output_names[0]) np_boxes = boxes_tensor.copy_to_cpu() t2 = time.time() diff --git a/deploy/python/predict_rec.py b/deploy/python/predict_rec.py index 0d0e7415996531a96bcb0f3b8b210f36fcd4eac8..e243358108edb7fe6f86ad9bd6a9f46c6b2c3040 100644 --- a/deploy/python/predict_rec.py +++ b/deploy/python/predict_rec.py @@ -58,12 +58,16 @@ class RecPredictor(Predictor): warmup=2) def predict(self, images, feature_normalize=True): - input_names = self.paddle_predictor.get_input_names() - input_tensor = self.paddle_predictor.get_input_handle(input_names[0]) + use_onnx = self.args.get("use_onnx", False) + if not use_onnx: + input_names = self.predictor.get_input_names() + input_tensor = self.predictor.get_input_handle(input_names[0]) - output_names = self.paddle_predictor.get_output_names() - output_tensor = self.paddle_predictor.get_output_handle(output_names[ - 0]) + output_names = self.predictor.get_output_names() + output_tensor = self.predictor.get_output_handle(output_names[0]) + else: + input_names = self.predictor.get_inputs()[0].name + output_names = self.predictor.get_outputs()[0].name if self.benchmark: self.auto_logger.times.start() @@ -76,9 +80,15 @@ class RecPredictor(Predictor): if self.benchmark: self.auto_logger.times.stamp() - input_tensor.copy_from_cpu(image) - self.paddle_predictor.run() - batch_output = output_tensor.copy_to_cpu() + if not use_onnx: + input_tensor.copy_from_cpu(image) + self.predictor.run() + batch_output = output_tensor.copy_to_cpu() + else: + batch_output = self.predictor.run( + output_names=[output_names], + input_feed={input_names: image})[0] + if self.benchmark: self.auto_logger.times.stamp() diff --git a/deploy/utils/predictor.py b/deploy/utils/predictor.py index 1c49c9dd5478932c655374fad541acbfc8952eeb..d44ae03df57def97bed6ab91205c327c3d318a2d 100644 --- a/deploy/utils/predictor.py +++ b/deploy/utils/predictor.py @@ -28,8 +28,12 @@ class Predictor(object): if args.use_fp16 is True: assert args.use_tensorrt is True self.args = args - self.paddle_predictor, self.config = self.create_paddle_predictor( - args, inference_model_dir) + if self.args.get("use_onnx", False): + self.predictor, self.config = self.create_onnx_predictor( + args, inference_model_dir) + else: + self.predictor, self.config = self.create_paddle_predictor( + args, inference_model_dir) def predict(self, image): raise NotImplementedError @@ -69,3 +73,20 @@ class Predictor(object): predictor = create_predictor(config) return predictor, config + + def create_onnx_predictor(self, args, inference_model_dir=None): + import onnxruntime as ort + if inference_model_dir is None: + inference_model_dir = args.inference_model_dir + model_file = os.path.join(inference_model_dir, "inference.onnx") + config = ort.SessionOptions() + if args.use_gpu: + raise ValueError( + "onnx inference now only supports cpu! please specify use_gpu false." + ) + else: + config.intra_op_num_threads = args.cpu_num_threads + if args.ir_optim: + config.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + predictor = ort.InferenceSession(model_file, sess_options=config) + return predictor, config diff --git a/test_tipc/config/ResNet/ResNet50_vd_linux_gpu_normal_normal_paddle2onnx_python_linux_cpu.txt b/test_tipc/config/ResNet/ResNet50_vd_linux_gpu_normal_normal_paddle2onnx_python_linux_cpu.txt new file mode 100644 index 0000000000000000000000000000000000000000..163bb48429b468433ee6bb539c029f51fe364190 --- /dev/null +++ b/test_tipc/config/ResNet/ResNet50_vd_linux_gpu_normal_normal_paddle2onnx_python_linux_cpu.txt @@ -0,0 +1,14 @@ +===========================paddle2onnx_params=========================== +model_name:ResNet50_vd +python:python3.7 +2onnx: paddle2onnx +--model_dir:./deploy/models/ResNet50_vd_infer/ +--model_filename:inference.pdmodel +--params_filename:inference.pdiparams +--save_file:./deploy/models/ResNet50_vd_infer/inference.onnx +--opset_version:10 +--enable_onnx_checker:True +inference: python/predict_cls.py -c configs/inference_cls.yaml +Global.use_onnx:True +Global.inference_model_dir:models/ResNet50_vd_infer/ +Global.use_gpu:False diff --git a/test_tipc/prepare.sh b/test_tipc/prepare.sh index 646e7f4d8dfef3f1bae4806985e037f9f1f5708c..89cdd505ed9f7781af00d8cffe35fff7cac21edb 100644 --- a/test_tipc/prepare.sh +++ b/test_tipc/prepare.sh @@ -165,3 +165,15 @@ if [ ${MODE} = "serving_infer" ];then cd ./deploy/paddleserving wget -nc https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/inference/ResNet50_vd_infer.tar && tar xf ResNet50_vd_infer.tar fi + +if [ ${MODE} = "paddle2onnx_infer" ];then + # prepare paddle2onnx env + python_name=$(func_parser_value "${lines[2]}") + ${python_name} -m pip install install paddle2onnx + ${python_name} -m pip install onnxruntime + + # wget model + cd deploy && mkdir models && cd models + wget -nc https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/inference/ResNet50_vd_infer.tar && tar xf ResNet50_vd_infer.tar + cd ../../ +fi diff --git a/test_tipc/test_paddle2onnx.sh b/test_tipc/test_paddle2onnx.sh index 300c61770d2519fad0502147e2cee4a3e4f50ac9..850fc9049b95400ee6334ff9dfa677947294c2de 100644 --- a/test_tipc/test_paddle2onnx.sh +++ b/test_tipc/test_paddle2onnx.sh @@ -11,7 +11,7 @@ python=$(func_parser_value "${lines[2]}") # parser params -dataline=$(awk 'NR==1, NR==12{print}' $FILENAME) +dataline=$(awk 'NR==1, NR==14{print}' $FILENAME) IFS=$'\n' lines=(${dataline}) @@ -33,12 +33,12 @@ enable_onnx_checker_key=$(func_parser_key "${lines[9]}") enable_onnx_checker_value=$(func_parser_value "${lines[9]}") # parser onnx inference inference_py=$(func_parser_value "${lines[10]}") -use_gpu_key=$(func_parser_key "${lines[11]}") -use_gpu_value=$(func_parser_value "${lines[11]}") -det_model_key=$(func_parser_key "${lines[12]}") -image_dir_key=$(func_parser_key "${lines[13]}") -image_dir_value=$(func_parser_value "${lines[13]}") - +use_onnx_key=$(func_parser_key "${lines[11]}") +use_onnx_value=$(func_parser_value "${lines[11]}") +inference_model_dir_key=$(func_parser_key "${lines[12]}") +inference_model_dir_value=$(func_parser_value "${lines[12]}") +inference_hardware_key=$(func_parser_key "${lines[13]}") +inference_hardware_value=$(func_parser_value "${lines[13]}") LOG_PATH="./test_tipc/output" mkdir -p ./test_tipc/output @@ -50,7 +50,7 @@ function func_paddle2onnx(){ _script=$1 # paddle2onnx - _save_log_path="${LOG_PATH}/paddle2onnx_infer_cpu.log" + _save_log_path=".${LOG_PATH}/paddle2onnx_infer_cpu.log" set_dirname=$(func_set_params "${infer_model_dir_key}" "${infer_model_dir_value}") set_model_filename=$(func_set_params "${model_filename_key}" "${model_filename_value}") set_params_filename=$(func_set_params "${params_filename_key}" "${params_filename_value}") @@ -62,10 +62,10 @@ function func_paddle2onnx(){ last_status=${PIPESTATUS[0]} status_check $last_status "${trans_model_cmd}" "${status_log}" # python inference - set_gpu=$(func_set_params "${use_gpu_key}" "${use_gpu_value}") - set_model_dir=$(func_set_params "${det_model_key}" "${save_file_value}") - set_img_dir=$(func_set_params "${image_dir_key}" "${image_dir_value}") - infer_model_cmd="${python} ${inference_py} ${set_gpu} ${set_img_dir} ${set_model_dir} --use_onnx=True > ${_save_log_path} 2>&1 " + set_model_dir=$(func_set_params "${inference_model_dir_key}" "${inference_model_dir_value}") + set_use_onnx=$(func_set_params "${use_onnx_key}" "${use_onnx_value}") + set_hardware=$(func_set_params "${inference_hardware_key}" "${inference_hardware_value}") + infer_model_cmd="cd deploy && ${python} ${inference_py} -o ${set_model_dir} -o ${set_use_onnx} -o ${set_hardware} >${_save_log_path} 2>&1 && cd ../" eval $infer_model_cmd status_check $last_status "${infer_model_cmd}" "${status_log}" }