From 4a3eb2c5f3c610cb259f80cfe191efef9edeb480 Mon Sep 17 00:00:00 2001 From: Junxian Ye Date: Sat, 12 Feb 2022 00:21:18 +0800 Subject: [PATCH] add support to fuse hardswish into 1 operation for ONNX (#1276) * fix upsample support error * add support for float type constant data loading * support opset 11 version of clip op convertion * support conv+clip fusion in graph_opt * add asymmetric transformation mode support for resize op * add hardswish fusion function support * add hardswish support Co-authored-by: yejunxian Co-authored-by: yejunxian --- tools/convert_tool/onnx/onnx2tengine.cpp | 4 +- tools/optimize/yolov5s-opt.py | 58 +++++++++++++++++++++--- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/tools/convert_tool/onnx/onnx2tengine.cpp b/tools/convert_tool/onnx/onnx2tengine.cpp index b0d26962..db45da4d 100644 --- a/tools/convert_tool/onnx/onnx2tengine.cpp +++ b/tools/convert_tool/onnx/onnx2tengine.cpp @@ -2132,7 +2132,9 @@ static int load_resize(ir_graph_t* graph, ir_node_t* node, const onnx::NodeProto interp_param->width_scale = 0; std::string coordinate_transformation_mode = GetAttributeOrDefault(onnx_node, "coordinate_transformation_mode", "half_pixel"); - TASSERT(coordinate_transformation_mode == "half_pixel" || coordinate_transformation_mode == "align_corners"); + TASSERT(coordinate_transformation_mode == "half_pixel" || + coordinate_transformation_mode == "align_corners" || + coordinate_transformation_mode == "asymmetric"); int align_corner = (coordinate_transformation_mode == "align_corners"); if (onnx_node.input_size() == 1) diff --git a/tools/optimize/yolov5s-opt.py b/tools/optimize/yolov5s-opt.py index 74f40cec..71f39852 100644 --- a/tools/optimize/yolov5s-opt.py +++ b/tools/optimize/yolov5s-opt.py @@ -22,7 +22,8 @@ https://github.com/ultralytics/yolov5 1. Remove the focus nodes of prepare process; 2. Remove the YOLO detection nodes of postprocess; 3. Fusion the activation HardSwish node replace the Sigmoid and Mul; -4. Update input/output tensor. +4. Fusion add+clip+div+mul ops into one HardSwish op; +5. Update input/output tensor. This tool is based on ONNX Framework. Usage: @@ -34,6 +35,7 @@ Author: xwwang@openailab.com, initial hhchen@openailab.com, update qinhj@lsec.cc.ac.cn, update + yejunxian@dm-ai.com, update """ import numpy as np @@ -45,14 +47,14 @@ from onnxsim import simplify def parse_args(): parser = argparse.ArgumentParser(description='YOLOv5 Optimize Tool Parameters') - - parser.add_argument('--input', help='input model path', default='./yolov5s.onnx', type=str) + + parser.add_argument('--input', help='input model path', default='./yolov5s.onnx', type=str) parser.add_argument('--output', help='output model path', default='./yolov5s-opt.onnx', type=str) parser.add_argument('--in_tensor', help='input tensor name', default='167', type=str) parser.add_argument('--out_tensor', help='output tensor names', default='381,420,459', type=str) parser.add_argument('--cut_focus', action='store_false', help='cut focus from model if true') parser.add_argument('--verbose', action='store_true', help='show verbose info') - + args = parser.parse_args() return args @@ -69,14 +71,14 @@ def cut_focus_output(input_node, in_name, out_name, cut_focus): out_name: output cut tensor value names Returns: new_nodes: the new node - """ + """ node_dict = {} # output node name for i in range(len(input_node)): node_dict[input_node[i].output[0]] = i if args.verbose: # (key, value): (output node[0] name, index) print("[Verbose] node_dict:", node_dict) - + # cut output nodes output_pass = np.zeros((len(input_node)), dtype=np.int) for i in range(len(out_name)): @@ -114,7 +116,7 @@ def fusion_hardswish(input_node): input_node: the nodes of ONNX model Returns: the new node - """ + """ del_list = [] for i in range(len(input_node) - 1): if (input_node[i].op_type == 'Sigmoid' and input_node[i+1].op_type == 'Mul'): @@ -127,6 +129,47 @@ def fusion_hardswish(input_node): return input_node +def fusion_hardswish_into_1op(input_node): + """ + using hardswish replace the add+clip+div+mul + + Args: + input_node: the nodes of ONNX model + Returns: + the new node + """ + new_nodes = [] + new_list = [] + tmp_list = [] + for i in range(len(input_node)): + if input_node[i].op_type == "Add" and \ + input_node[i+1].op_type == "Clip" and \ + input_node[i+2].op_type == "Div" and \ + input_node[i+3].op_type == "Mul": + suffix = input_node[i+3].name.split("_")[-1] + new_node = onnx.helper.make_node( + name=f"hardswish_{suffix}", + inputs=input_node[i].input[:1], + outputs=input_node[i+3].output, + op_type="Sigmoid" + ) + new_node.op_type = "HardSwish" + new_nodes.append(new_node) + tmp_list.append(None) + tmp_list.append(input_node[i]) + + # in order to keep tapology sort order + i = 0 + while i < len(tmp_list): + if tmp_list[i] == None: + new_list.append(new_nodes.pop(0)) + i += 5 # skip add+clip+div+mul + else: + new_list.append(tmp_list[i]) + i += 1 + + return new_list + def keep_or_del_elem(obj, elem_name_list, keep=False): """ @@ -190,6 +233,7 @@ def main(): # op fusion, using HardSwish replace the Sigmoid and Mul print("[Quant Tools Info]: Step 2, Using hardswish replace the sigmoid and mul.") new_nodes = fusion_hardswish(new_nodes) + new_nodes = fusion_hardswish_into_1op(new_nodes) # rebuild new model, set the input and outputs nodes print("[Quant Tools Info]: Step 3, Rebuild onnx graph nodes.") -- GitLab