diff --git a/configs/rotate/ppyoloe_r/README.md b/configs/rotate/ppyoloe_r/README.md index c709fec6fe0fee6053f4af6a9d18d79fcbd69915..f2fae8bbb011aace337e22712309265683cd836d 100644 --- a/configs/rotate/ppyoloe_r/README.md +++ b/configs/rotate/ppyoloe_r/README.md @@ -123,6 +123,21 @@ python deploy/python/infer.py --image_file demo/P0072__1.0__0___0.png --model_di **注意:** - 使用Paddle-TRT使用确保**PaddlePaddle版本为develop版本且TensorRT版本大于8.2**. +**使用ONNX Runtime进行部署**,执行以下命令: +``` +# 导出模型 +python tools/export_model.py -c configs/rotate/ppyoloe_r/ppyoloe_r_crn_l_3x_dota.yml -o weights=https://paddledet.bj.bcebos.com/models/ppyoloe_r_crn_l_3x_dota.pdparams export_onnx=True + +# 安装paddle2onnx +pip install paddle2onnx + +# 转换成onnx模型 +paddle2onnx --model_dir output_inference/ppyoloe_r_crn_l_3x_dota --model_filename model.pdmodel --params_filename model.pdiparams --opset_version 11 --save_file ppyoloe_r_crn_l_3x_dota.onnx + +# 预测图片 +python configs/rotate/tools/onnx_infer.py --infer_cfg output_inference/ppyoloe_r_crn_l_3x_dota/infer_cfg.yml --onnx_file ppyoloe_r_crn_l_3x_dota.onnx --image_file demo/P0072__1.0__0___0.png + +``` ## 附录 diff --git a/configs/rotate/ppyoloe_r/README_en.md b/configs/rotate/ppyoloe_r/README_en.md index 3301bf8061f98f2d27790e6abbc215b2c6032b42..85876c5c54ee75fdd862604387f5ef49beada2af 100644 --- a/configs/rotate/ppyoloe_r/README_en.md +++ b/configs/rotate/ppyoloe_r/README_en.md @@ -114,7 +114,7 @@ python tools/export_model.py -c configs/rotate/ppyoloe_r/ppyoloe_r_crn_l_3x_dota python deploy/python/infer.py --image_file demo/P0072__1.0__0___0.png --model_dir=output_inference/ppyoloe_r_crn_l_3x_dota --run_mode=paddle --device=gpu ``` -**Using Paddle-TRT** to for deployment, run following command +**Using Paddle-TRT** for deployment, run following command ``` bash # export inference model @@ -126,6 +126,22 @@ python deploy/python/infer.py --image_file demo/P0072__1.0__0___0.png --model_di **Notes:** - When using Paddle-TRT for speed testing, make sure that **the version of TensorRT is larger than 8.2 and the version of PaddlePaddle is the develop version** +**Using ONNX Runtime** for deployment, run following command + +``` bash +# export inference model +python tools/export_model.py -c configs/rotate/ppyoloe_r/ppyoloe_r_crn_l_3x_dota.yml -o weights=https://paddledet.bj.bcebos.com/models/ppyoloe_r_crn_l_3x_dota.pdparams export_onnx=True + +# install paddle2onnx +pip install paddle2onnx + +# convert to onnx model +paddle2onnx --model_dir output_inference/ppyoloe_r_crn_l_3x_dota --model_filename model.pdmodel --params_filename model.pdiparams --opset_version 11 --save_file ppyoloe_r_crn_l_3x_dota.onnx + +# inference single image +python configs/rotate/tools/onnx_infer.py --infer_cfg output_inference/ppyoloe_r_crn_l_3x_dota/infer_cfg.yml --onnx_file ppyoloe_r_crn_l_3x_dota.onnx --image_file demo/P0072__1.0__0___0.png +``` + ## Appendix Ablation experiments of PP-YOLOE-R diff --git a/configs/rotate/tools/onnx_infer.py b/configs/rotate/tools/onnx_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..fa9b06d2ddd525ed32f99891bdd67f3b6650f0be --- /dev/null +++ b/configs/rotate/tools/onnx_infer.py @@ -0,0 +1,302 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import six +import glob +import copy +import yaml +import argparse +import cv2 +import numpy as np +from shapely.geometry import Polygon +from onnxruntime import InferenceSession + + +# preprocess ops +def decode_image(img_path): + with open(img_path, 'rb') as f: + im_read = f.read() + data = np.frombuffer(im_read, dtype='uint8') + im = cv2.imdecode(data, 1) # BGR mode, but need RGB mode + im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) + img_info = { + "im_shape": np.array( + im.shape[:2], dtype=np.float32), + "scale_factor": np.array( + [1., 1.], dtype=np.float32) + } + return im, img_info + + +class Resize(object): + def __init__(self, target_size, keep_ratio=True, interp=cv2.INTER_LINEAR): + if isinstance(target_size, int): + target_size = [target_size, target_size] + self.target_size = target_size + self.keep_ratio = keep_ratio + self.interp = interp + + def __call__(self, im, im_info): + assert len(self.target_size) == 2 + assert self.target_size[0] > 0 and self.target_size[1] > 0 + im_channel = im.shape[2] + im_scale_y, im_scale_x = self.generate_scale(im) + im = cv2.resize( + im, + None, + None, + fx=im_scale_x, + fy=im_scale_y, + interpolation=self.interp) + im_info['im_shape'] = np.array(im.shape[:2]).astype('float32') + im_info['scale_factor'] = np.array( + [im_scale_y, im_scale_x]).astype('float32') + return im, im_info + + def generate_scale(self, im): + origin_shape = im.shape[:2] + im_c = im.shape[2] + if self.keep_ratio: + im_size_min = np.min(origin_shape) + im_size_max = np.max(origin_shape) + target_size_min = np.min(self.target_size) + target_size_max = np.max(self.target_size) + im_scale = float(target_size_min) / float(im_size_min) + if np.round(im_scale * im_size_max) > target_size_max: + im_scale = float(target_size_max) / float(im_size_max) + im_scale_x = im_scale + im_scale_y = im_scale + else: + resize_h, resize_w = self.target_size + im_scale_y = resize_h / float(origin_shape[0]) + im_scale_x = resize_w / float(origin_shape[1]) + return im_scale_y, im_scale_x + + +class Permute(object): + def __init__(self, ): + super(Permute, self).__init__() + + def __call__(self, im, im_info): + im = im.transpose((2, 0, 1)) + return im, im_info + + +class NormalizeImage(object): + def __init__(self, mean, std, is_scale=True, norm_type='mean_std'): + self.mean = mean + self.std = std + self.is_scale = is_scale + self.norm_type = norm_type + + def __call__(self, im, im_info): + im = im.astype(np.float32, copy=False) + if self.is_scale: + scale = 1.0 / 255.0 + im *= scale + + if self.norm_type == 'mean_std': + mean = np.array(self.mean)[np.newaxis, np.newaxis, :] + std = np.array(self.std)[np.newaxis, np.newaxis, :] + im -= mean + im /= std + return im, im_info + + +class PadStride(object): + def __init__(self, stride=0): + self.coarsest_stride = stride + + def __call__(self, im, im_info): + coarsest_stride = self.coarsest_stride + if coarsest_stride <= 0: + return im, im_info + im_c, im_h, im_w = im.shape + pad_h = int(np.ceil(float(im_h) / coarsest_stride) * coarsest_stride) + pad_w = int(np.ceil(float(im_w) / coarsest_stride) * coarsest_stride) + padding_im = np.zeros((im_c, pad_h, pad_w), dtype=np.float32) + padding_im[:, :im_h, :im_w] = im + return padding_im, im_info + + +class Compose: + def __init__(self, transforms): + self.transforms = [] + for op_info in transforms: + new_op_info = op_info.copy() + op_type = new_op_info.pop('type') + self.transforms.append(eval(op_type)(**new_op_info)) + + def __call__(self, img_path): + img, im_info = decode_image(img_path) + for t in self.transforms: + img, im_info = t(img, im_info) + inputs = copy.deepcopy(im_info) + inputs['image'] = img + return inputs + + +# postprocess +def rbox_iou(g, p): + g = np.array(g) + p = np.array(p) + g = Polygon(g[:8].reshape((4, 2))) + p = Polygon(p[:8].reshape((4, 2))) + g = g.buffer(0) + p = p.buffer(0) + if not g.is_valid or not p.is_valid: + return 0 + inter = Polygon(g).intersection(Polygon(p)).area + union = g.area + p.area - inter + if union == 0: + return 0 + else: + return inter / union + + +def multiclass_nms_rotated(pred_bboxes, + pred_scores, + iou_threshlod=0.1, + score_threshold=0.1): + """ + Args: + pred_bboxes (numpy.ndarray): [B, N, 8] + pred_scores (numpy.ndarray): [B, C, N] + + Return: + bboxes (numpy.ndarray): [N, 10] + bbox_num (numpy.ndarray): [B] + """ + bbox_num = [] + bboxes = [] + for bbox_per_img, score_per_img in zip(pred_bboxes, pred_scores): + num_per_img = 0 + for cls_id, score_per_cls in enumerate(score_per_img): + keep_mask = score_per_cls > score_threshold + bbox = bbox_per_img[keep_mask] + score = score_per_cls[keep_mask] + + idx = score.argsort()[::-1] + bbox = bbox[idx] + score = score[idx] + keep_idx = [] + for i, b in enumerate(bbox): + supressed = False + for gi in keep_idx: + g = bbox[gi] + if rbox_iou(b, g) > iou_threshlod: + supressed = True + break + + if supressed: + continue + + keep_idx.append(i) + + keep_box = bbox[keep_idx] + keep_score = score[keep_idx] + keep_cls_ids = np.ones(len(keep_idx)) * cls_id + bboxes.append( + np.concatenate( + [keep_cls_ids[:, None], keep_score[:, None], keep_box], + axis=-1)) + num_per_img += len(keep_idx) + + bbox_num.append(num_per_img) + + return np.concatenate(bboxes, axis=0), np.array(bbox_num) + + +def get_test_images(infer_dir, infer_img): + """ + Get image path list in TEST mode + """ + assert infer_img is not None or infer_dir is not None, \ + "--image_file or --image_dir should be set" + assert infer_img is None or os.path.isfile(infer_img), \ + "{} is not a file".format(infer_img) + assert infer_dir is None or os.path.isdir(infer_dir), \ + "{} is not a directory".format(infer_dir) + + # infer_img has a higher priority + if infer_img and os.path.isfile(infer_img): + return [infer_img] + + images = set() + infer_dir = os.path.abspath(infer_dir) + assert os.path.isdir(infer_dir), \ + "infer_dir {} is not a directory".format(infer_dir) + exts = ['jpg', 'jpeg', 'png', 'bmp'] + exts += [ext.upper() for ext in exts] + for ext in exts: + images.update(glob.glob('{}/*.{}'.format(infer_dir, ext))) + images = list(images) + + assert len(images) > 0, "no image found in {}".format(infer_dir) + print("Found {} inference images in total.".format(len(images))) + + return images + + +def predict_image(infer_config, predictor, img_list): + # load preprocess transforms + transforms = Compose(infer_config['Preprocess']) + # predict image + for img_path in img_list: + inputs = transforms(img_path) + inputs_name = [var.name for var in predictor.get_inputs()] + inputs = {k: inputs[k][None, ] for k in inputs_name} + + outputs = predictor.run(output_names=None, input_feed=inputs) + + bboxes, bbox_num = multiclass_nms_rotated( + np.array(outputs[0]), np.array(outputs[1])) + print("ONNXRuntime predict: ") + for bbox in bboxes: + if bbox[0] > -1 and bbox[1] > infer_config['draw_threshold']: + print(f"{int(bbox[0])} {bbox[1]} " + f"{bbox[2]} {bbox[3]} {bbox[4]} {bbox[5]}" + f"{bbox[6]} {bbox[7]} {bbox[8]} {bbox[9]}") + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--infer_cfg", type=str, help="infer_cfg.yml") + parser.add_argument( + '--onnx_file', + type=str, + default="model.onnx", + help="onnx model file path") + parser.add_argument("--image_dir", type=str) + parser.add_argument("--image_file", type=str) + return parser.parse_args() + + +if __name__ == '__main__': + FLAGS = parse_args() + # load image list + img_list = get_test_images(FLAGS.image_dir, FLAGS.image_file) + # load predictor + predictor = InferenceSession(FLAGS.onnx_file) + # load infer config + with open(FLAGS.infer_cfg) as f: + infer_config = yaml.safe_load(f) + + predict_image(infer_config, predictor, img_list) diff --git a/ppdet/modeling/heads/ppyoloe_r_head.py b/ppdet/modeling/heads/ppyoloe_r_head.py index 89cb0fa82a07085154665e45680460551e46fb8b..aaf21063204a0c5667f5edefaa31485ddfc6c1cc 100644 --- a/ppdet/modeling/heads/ppyoloe_r_head.py +++ b/ppdet/modeling/heads/ppyoloe_r_head.py @@ -44,7 +44,7 @@ class ESEAttn(nn.Layer): @register class PPYOLOERHead(nn.Layer): - __shared__ = ['num_classes', 'trt'] + __shared__ = ['num_classes', 'trt', 'export_onnx'] __inject__ = ['static_assigner', 'assigner', 'nms'] def __init__(self, @@ -57,6 +57,7 @@ class PPYOLOERHead(nn.Layer): use_varifocal_loss=True, static_assigner_epoch=4, trt=False, + export_onnx=False, static_assigner='ATSSAssigner', assigner='TaskAlignedAssigner', nms='MultiClassNMS', @@ -84,6 +85,8 @@ class PPYOLOERHead(nn.Layer): self.stem_cls = nn.LayerList() self.stem_reg = nn.LayerList() self.stem_angle = nn.LayerList() + trt = False if export_onnx else trt + self.export_onnx = export_onnx act = get_act_fn( act, trt=trt) if act is None or isinstance(act, (str, dict)) else act @@ -415,5 +418,7 @@ class PPYOLOERHead(nn.Layer): ], axis=-1).reshape([-1, 1, 8]) pred_bboxes /= scale_factor + if self.export_onnx: + return pred_bboxes, pred_scores bbox_pred, bbox_num, _ = self.nms(pred_bboxes, pred_scores) return bbox_pred, bbox_num