support Clip op conversion with onnx opset version 11 (#1245)

* fix upsample support error

* add support for float type constant data loading

* support opset 11 version of clip op convertion

* support conv+clip fusion in graph_opt
Co-authored-by: Nyejunxian <yejunxian@dm-ai.com>

tools/convert_tool/onnx/onnx2tengine.cpp

@@ -318,106 +318,113 @@ int onnx_serializer::load_constant_tensor(ir_graph_t* graph, const onnx::GraphPr
 
         const std::string& op = node.op_type();
 
-        if ((op == "Reshape" || op == "Gather" || op == "Div" || op == "Resize" || op == "Upsample"))
+        if ((op == "Reshape" || op == "Gather" ||
+            op == "Div" || op == "Resize" ||
+            op == "Upsample" || op == "Clip") &&
+            (node.input_size() > 1))
         {
-            if (node_tensor.count(node.input(1)) == 0)
-                continue;
-            const onnx::TensorProto& onnx_tensor = node_tensor[node.input(1)];
-            std::pair<std::string, bool> t(node.input(1), 0);
-            tensor_check.insert(t);
-            int tensor_data_type = get_onnx_tensor_data_type(onnx_tensor);
-            if (tensor_data_type < 0)
+            // iter over constant inputs and create ir_tensor for constant tensor
+            for(int inp_idx = 0; inp_idx < node.input_size(); ++inp_idx)
             {
-                return -1;
-            }
+                if (node_tensor.count(node.input(inp_idx)) == 0)
+                    continue;
+                const onnx::TensorProto& onnx_tensor = node_tensor[node.input(inp_idx)];
+                std::pair<std::string, bool> t(node.input(inp_idx), 0);
+                tensor_check.insert(t);
+                int tensor_data_type = get_onnx_tensor_data_type(onnx_tensor);
+                if (tensor_data_type < 0)
+                {
+                    return -1;
+                }
 
-            const char* name = node.input(1).c_str();
-            int dim_num = onnx_tensor.dims_size();
-            std::vector<int> dims(dim_num);
-            for (int j = 0; j < dim_num; j++)
-            {
-                dims[j] = onnx_tensor.dims(j);
-            }
+                const char* name = node.input(inp_idx).c_str();
+                int dim_num = onnx_tensor.dims_size();
+                std::vector<int> dims(dim_num);
+                for (int j = 0; j < dim_num; j++)
+                {
+                    dims[j] = onnx_tensor.dims(j);
+                }
 
-            // create ir tensor
-            ir_tensor_t* ir_tensor = create_ir_tensor(graph, name, tensor_data_type);
-            if (ir_tensor == NULL)
-            {
-                fprintf(stderr, "create ir tensor failed!\n");
-                return -1;
-            }
-            set_ir_tensor_shape(ir_tensor, dims.data(), dim_num);
-            ir_tensor->tensor_type = TENSOR_TYPE_CONST;
-            // set tensor data
-            if (7 == onnx_tensor.data_type())
-            {
-                int tensor_size = ir_tensor->elem_num * sizeof(int64_t);
-                ir_tensor->data = sys_malloc(tensor_size);
-                int64_t* mem_buf = (int64_t*)ir_tensor->data;
-                if (onnx_tensor.has_raw_data())
+                // create ir tensor
+                ir_tensor_t* ir_tensor = create_ir_tensor(graph, name, tensor_data_type);
+                if (ir_tensor == NULL)
                 {
-                    int64_t* raw_data = (int64_t*)onnx_tensor.raw_data().data();
-                    for (int j = 0; j < ir_tensor->elem_num; j++)
-                    {
-                        mem_buf[j] = raw_data[j];
-                    }
+                    fprintf(stderr, "create ir tensor failed!\n");
+                    return -1;
                 }
-                else
+                set_ir_tensor_shape(ir_tensor, dims.data(), dim_num);
+                ir_tensor->tensor_type = TENSOR_TYPE_CONST;
+                // set tensor data
+                if (7 == onnx_tensor.data_type())
                 {
-                    int64_t* raw_data = (int64_t*)onnx_tensor.int64_data().data();
-                    for (int j = 0; j < ir_tensor->elem_num; j++)
+                    int tensor_size = ir_tensor->elem_num * sizeof(int64_t);
+                    ir_tensor->data = sys_malloc(tensor_size);
+                    int64_t* mem_buf = (int64_t*)ir_tensor->data;
+                    if (onnx_tensor.has_raw_data())
                     {
-                        mem_buf[j] = raw_data[j];
+                        int64_t* raw_data = (int64_t*)onnx_tensor.raw_data().data();
+                        for (int j = 0; j < ir_tensor->elem_num; j++)
+                        {
+                            mem_buf[j] = raw_data[j];
+                        }
                     }
-                }
-            }
-            else if(tensor_data_type == TENGINE_DT_FP32)
-            {
-                // to support float type constant data loading
-                int tensor_size = ir_tensor->elem_num * sizeof(float_t);
-                ir_tensor->data = sys_malloc(tensor_size);
-                float_t* mem_buf = (float_t*)ir_tensor->data;
-                if (onnx_tensor.has_raw_data())
-                {
-                    float_t* raw_data = (float_t*)onnx_tensor.raw_data().data();
-                    for (int j = 0; j < ir_tensor->elem_num; j++)
+                    else
                     {
-                        mem_buf[j] = raw_data[j];
+                        int64_t* raw_data = (int64_t*)onnx_tensor.int64_data().data();
+                        for (int j = 0; j < ir_tensor->elem_num; j++)
+                        {
+                            mem_buf[j] = raw_data[j];
+                        }
                     }
                 }
-                else
+                else if(tensor_data_type == TENGINE_DT_FP32)
                 {
-                    int32_t* raw_data = (int32_t*)onnx_tensor.int32_data().data();
-                    for (int j = 0; j < ir_tensor->elem_num; j++)
+                    // to support float type constant data loading
+                    int tensor_size = ir_tensor->elem_num * sizeof(float_t);
+                    ir_tensor->data = sys_malloc(tensor_size);
+                    float_t* mem_buf = (float_t*)ir_tensor->data;
+                    if (onnx_tensor.has_raw_data())
                     {
-                        mem_buf[j] = raw_data[j];
+                        float_t* raw_data = (float_t*)onnx_tensor.raw_data().data();
+                        for (int j = 0; j < ir_tensor->elem_num; j++)
+                        {
+                            mem_buf[j] = raw_data[j];
+                        }
                     }
-                }
-            }
-            else
-            {
-                int tensor_size = ir_tensor->elem_num * sizeof(uint8_t);
-                ir_tensor->data = sys_malloc(tensor_size);
-                uint8_t* mem_buf = (uint8_t*)ir_tensor->data;
-                if (onnx_tensor.has_raw_data())
-                {
-                    uint8_t* raw_data = (uint8_t*)onnx_tensor.raw_data().data();
-                    for (int j = 0; j < ir_tensor->elem_num; j++)
+                    else
                     {
-                        mem_buf[j] = raw_data[j];
+                        int32_t* raw_data = (int32_t*)onnx_tensor.int32_data().data();
+                        for (int j = 0; j < ir_tensor->elem_num; j++)
+                        {
+                            mem_buf[j] = raw_data[j];
+                        }
                     }
                 }
                 else
                 {
-                    uint8_t* raw_data = (uint8_t*)onnx_tensor.int32_data().data();
-                    for (int j = 0; j < ir_tensor->elem_num; j++)
+                    int tensor_size = ir_tensor->elem_num * sizeof(uint8_t);
+                    ir_tensor->data = sys_malloc(tensor_size);
+                    uint8_t* mem_buf = (uint8_t*)ir_tensor->data;
+                    if (onnx_tensor.has_raw_data())
+                    {
+                        uint8_t* raw_data = (uint8_t*)onnx_tensor.raw_data().data();
+                        for (int j = 0; j < ir_tensor->elem_num; j++)
+                        {
+                            mem_buf[j] = raw_data[j];
+                        }
+                    }
+                    else
                     {
-                        mem_buf[j] = raw_data[j];
+                        uint8_t* raw_data = (uint8_t*)onnx_tensor.int32_data().data();
+                        for (int j = 0; j < ir_tensor->elem_num; j++)
+                        {
+                            mem_buf[j] = raw_data[j];
+                        }
                     }
                 }
+                ir_node_t* ir_node = create_ir_node(graph, name, OP_CONST, OP_VERSION);
+                set_ir_node_output_tensor(ir_node, 0, ir_tensor);
             }
-            ir_node_t* ir_node = create_ir_node(graph, name, OP_CONST, OP_VERSION);
-            set_ir_node_output_tensor(ir_node, 0, ir_tensor);
         }
     }
 

tools/convert_tool/utils/graph_optimizer/graph_opt.cpp

@@ -628,7 +628,9 @@ static int fuse_conv_relu_common(ir_graph_t* graph)
     for (size_t i = 0; i < graph->node_num; i++)
     {
         ir_node_t* relu_node = get_ir_graph_node(graph, i);
-        if (relu_node->op.type != OP_RELU && relu_node->op.type != OP_RELU6)
+        if (relu_node->op.type != OP_RELU &&
+            relu_node->op.type != OP_RELU6 &&
+            relu_node->op.type != OP_CLIP)
             continue;
         if (relu_node->op.type == OP_RELU)
         {
@@ -636,6 +638,12 @@ static int fuse_conv_relu_common(ir_graph_t* graph)
             if (relu_param->negative_slope != 0.f)
                 continue;
         }
+        if (relu_node->op.type == OP_CLIP)
+        {
+            struct clip_param* clip_param = (struct clip_param*)relu_node->op.param_mem;
+            if (clip_param->min != 0.f && clip_param->max != 6.f)
+                continue;
+        }
         ir_tensor_t* conv_tensor = get_ir_graph_tensor(graph, relu_node->input_tensors[0]);
         ir_node_t* conv_node = get_ir_graph_node(graph, conv_tensor->producer);
         if (conv_node->op.type != OP_CONV)
@@ -654,7 +662,8 @@ static int fuse_conv_relu_common(ir_graph_t* graph)
         struct conv_param* conv_param = (struct conv_param*)conv_node->op.param_mem;
         if (relu_node->op.type == OP_RELU)
             conv_param->activation = 0;
-        if (relu_node->op.type == OP_RELU6)
+        if (relu_node->op.type == OP_RELU6 ||
+            relu_node->op.type == OP_CLIP)
             conv_param->activation = 6;
 
         /* delete relu node */

tools/convert_tool/utils/graph_optimizer/graph_opt.hpp

@@ -21,6 +21,7 @@ extern "C" {
 #include "eltwise_param.h"
 #include "batchnorm_param.h"
 #include "fc_param.h"
+#include "clip_param.h"
 }
 
 int graph_opt(graph_t graph);