Merge pull request #458 from SunAhong1993/paddle-2.0

onnx static code 

x2paddle/convert.py

@@ -185,16 +185,8 @@ def onnx2paddle(model_path, save_dir, paddle_type, params_merge=False):
         from x2paddle.op_mapper.static.onnx2paddle.onnx_op_mapper import ONNXOpMapper
     model = ONNXDecoder(model_path)
     mapper = ONNXOpMapper(model)
-    if paddle_type == "dygraph":
-        mapper.paddle_graph.build()
-        mapper.paddle_graph.gen_model(save_dir)
-    else:
-        from x2paddle.optimizer.onnx_optimizer import ONNXOptimizer
-        print("Model optimizing ...")
-        optimizer = ONNXOptimizer(mapper)
-        optimizer.delete_redundance_code()
-        print("Model optimized.")
-        mapper.save_inference_model(save_dir, params_merge)
+    mapper.paddle_graph.build()
+    mapper.paddle_graph.gen_model(save_dir)
 
 
 def pytorch2paddle(module, save_dir, jit_type="trace", input_examples=None):

x2paddle/decoder/onnx_decoder.py

@@ -117,7 +117,13 @@ class ONNXGraphDataNode(GraphNode):
         if isinstance(self.layer, ValueInfoProto):
             values = self.layer.type.tensor_type.shape.dim
             out_shapes = list()
-            out_shapes.append([dim.dim_value for dim in values])
+            shape = list()
+            for dim in values:
+                if dim.dim_value == 0:
+                    shape.append(-1)
+                else:
+                    shape.append(dim.dim_value)
+            out_shapes.append(shape)
             return out_shapes
         else:
             values = self.layer.dims

x2paddle/op_mapper/dygraph/onnx2paddle/opset9/opset.py

@@ -534,7 +534,7 @@ class OpSet9():
             'bias_attr': string(val_b.name)
         }
         dim = len(val_x.out_shapes[0])
-        if dim == 2 or dim == 3:
+        if dim == 3:
             paddle_op = "paddle.nn.InstanceNorm1D"
         elif dim == 4:
             paddle_op = "paddle.nn.InstanceNorm2D"
@@ -1539,7 +1539,6 @@ class OpSet9():
         layer_outputs = [op_name, output_name]
         val_x = self.graph.get_input_node(node, idx=0, copy=True)
         val_w = self.graph.get_input_node(node, idx=1, copy=True)
-        val_y = self.graph.get_node(node.layer.output[0], copy=True)
         has_bias = len(node.layer.input) == 3
         if has_bias:
             val_b = self.graph.get_input_node(node, idx=2, copy=True)
@@ -1620,23 +1619,7 @@ class OpSet9():
         output_size[1] = (val_x.out_shapes[0][3] - 1
                           ) * strides[1] - 2 * paddings[1] + dilations[1] * (
                               kernel_shape[1] - 1) + 1 + out_padding[1]
-#         layer_attrs = {
-#             'in_channels': num_in_channels,
-#             'out_channels': num_out_channels,
-#             'output_size': output_size or None,
-#             'kernel_size': kernel_shape,
-#             'padding': paddings,
-#             'stride': strides,
-#             'dilation': dilations,
-#             'groups': num_groups,
-#             'weight_attr': string(val_w.name),
-#             'bias_attr': None if val_b is None else string(val_b.name),
-#         }
-#         self.paddle_graph.add_layer(
-#             paddle_op, 
-#             inputs={"x": val_x.name}, 
-#             outputs=layer_outputs, 
-#             **layer_attrs)
+        # Conv2DTranspose缺少output_size，只能在forward里头传进output_size
         inputs_dict = {'x': val_x if isinstance(val_x, str) else val_x.name,
                        "weight": val_w.name}
         layer_attrs = {

x2paddle/op_mapper/static/onnx2paddle/onnx_op_mapper.py

@@ -12,9 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from x2paddle.op_mapper.static.onnx2paddle.opset9 import OpSet9, custom_layers
+import sys
+from x2paddle.op_mapper.static.onnx2paddle.opset9 import OpSet9
 from x2paddle.core.op_mapper import OpMapper
-from x2paddle.decoder.onnx_decoder import ONNXGraph, ONNXGraphNode, ONNXGraphDataNode
+from x2paddle.decoder.onnx_decoder import ONNXGraphNode
+from x2paddle.core.program import PaddleGraph
 
 
 class ONNXOpMapper(OpMapper):
@@ -23,33 +25,36 @@ class ONNXOpMapper(OpMapper):
         self.support_op_sets = [9, ]
         self.default_op_set = 9
         self.graph = decoder.graph
+        self.paddle_graph = PaddleGraph(parent_layer=None, graph_type="static", source_type="onnx")
+        self.paddle_graph.outputs = self.graph.output_nodes
         self.opset = self.create_opset(decoder)
         if not self.op_checker():
-            raise Exception("Model are not supported yet.")
-        #mapping op
+            raise Exception("Model is not supported yet.")
+        
         print("Total nodes: {}".format(
             sum([
                 isinstance(node, ONNXGraphNode)
                 for name, node in self.graph.node_map.items()
             ])))
-
         print("Nodes converting ...")
-        for node_name in self.graph.topo_sort:
+        for i, node_name in enumerate(self.graph.topo_sort):
+            sys.stderr.write("\rConverting node {} ...     ".format(i + 1))
             node = self.graph.get_node(node_name)
             op = node.layer_type
             if hasattr(self.opset, op):
                 func = getattr(self.opset, op)
                 func(node)
-            elif op in self.opset.default_op_mapping:
+            elif op in self.opset.directly_map_ops:
                 self.opset.directly_map(node)
-            elif op in custom_layers:
-                self.opset.deal_custom_layer(node)
             elif op in self.opset.elementwise_ops:
                 self.opset.elementwise_map(node)
-        print("Nodes converted.")
-        self.weights = self.opset.weights
-        self.omit_nodes = self.opset.omit_nodes
-        self.used_custom_layers = self.opset.used_custom_layers
+        print("\nNodes converted.")
+        self.paddle_graph.set_name(self.graph.graph_name)
+        self.paddle_graph.set_parameters(self.opset.params)
+        self.paddle_graph.set_inputs_info(self.opset.inputs_info)
+        self.paddle_graph.inputs = self.graph.input_nodes
+        self.paddle_graph.outputs = self.graph.output_nodes
+        
 
     def op_checker(self):
         unsupported_ops = set()
@@ -57,17 +62,17 @@ class ONNXOpMapper(OpMapper):
             node = self.graph.get_node(node_name)
             op = node.layer_type
             if not hasattr(self.opset, op) and \
-                op not in self.opset.default_op_mapping and \
-                op not in custom_layers and \
+                op not in self.opset.directly_map_ops and \
                 op not in self.opset.elementwise_ops:
                 unsupported_ops.add(op)
         if len(unsupported_ops) == 0:
             return True
         else:
-            print("There are {} ops not supported yet, list as below".format(
-                len(unsupported_ops)))
+            if len(unsupported_ops) > 0:
+                print("\n========= {} OPs are not supported yet ===========".format(
+                    len(unsupported_ops)))
             for op in unsupported_ops:
-                print(op)
+                print("========== {} ============".format(op))
             return False
 
     def create_opset(self, decoder):
@@ -88,4 +93,4 @@ class ONNXOpMapper(OpMapper):
             'Now, onnx2paddle support convert onnx model opset_verison {},'
             'opset_verison of your onnx model is {}, automatically treated as op_set: {}.'
             .format(self.support_op_sets, decoder.op_set, run_op_set))
-        return eval(opset)(decoder)
+        return eval(opset)(decoder, self.paddle_graph)

x2paddle/op_mapper/static/onnx2paddle/opset9/__init__.py

 from .opset import OpSet9
-from .custom_layer import custom_layers

x2paddle/op_mapper/static/onnx2paddle/opset9/custom_layer/__init__.py

-#   Copyright (c) 2019  PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from .register import get_registered_layers
-
-custom_layers = get_registered_layers()
-
-
-def set_args(f, params):
-    """ set args for function 'f' using the parameters in node.layer.param
-    Args:
-        f (function): a python function object
-        params (object): a object contains attributes needed by f's arguments
-    Returns:
-        arg_names (list): a list of argument names
-        kwargs (dict): a dict contains needed arguments
-    """
-    argc = f.__code__.co_argcount
-    arg_list = f.__code__.co_varnames[0:argc]
-    kwargs = {}
-    for arg_name in arg_list:
-        if hasattr(params, arg_name) and params is not None:
-            kwargs[arg_name] = getattr(params, arg_name)
-    return arg_list, kwargs
-
-
-def has_layer(layer_type):
-    """ test whether this layer exists in custom layer
-    """
-    return layer_type in custom_layers
-
-
-def get_params(layer, layer_type):
-    import re
-    if layer_type.lower() == "deconvolution" or layer_type.lower(
-    ) == "convolutiondepthwise":
-        param_name = '_'.join(('convolution', 'param'))
-    elif layer_type.lower() == "normalize":
-        param_name = '_'.join(('norm', 'param'))
-    elif len(layer_type) - len(re.sub("[A-Z]", "", layer_type)) >= 2:
-        s = ''
-        tmp_name = ''
-        for i, ch in enumerate(layer_type):
-            if i == 0:
-                s += ch.lower()
-                continue
-            elif ch.isupper() and layer_type[i - 1].islower():
-                tmp_name += (s + '_')
-                s = ''
-            s += ch.lower()
-        tmp_name += s
-        param_name = '_'.join((tmp_name, 'param'))
-    else:
-        param_name = '_'.join((layer_type.lower(), 'param'))
-    return getattr(layer, param_name, None)
-
-
-def compute_output_shape(node):
-    """ compute the output shape of custom layer
-    """
-    layer_type = node.layer_type
-    assert layer_type in custom_layers, "layer[%s] not exist in custom layers" % (
-        layer_type)
-    shape_func = custom_layers[layer_type]['shape']
-    layer = node.layer
-    params = get_params(layer, layer_type)
-    arg_names, kwargs = set_args(shape_func, params)
-    input_shape = node.input_shape
-    return shape_func(input_shape, **kwargs)
-
-
-def make_custom_layer(node):
-    """ get the code which implement the custom layer function
-    """
-    layer_type = node.layer_type
-    assert layer_type in custom_layers, "layer[%s] not exist in custom layers" % (
-        layer_type)
-    layer_func = custom_layers[layer_type]['layer']
-    import inspect
-    return inspect.getsource(layer_func), layer_func
-
-
-def make_custom_child_func(node):
-    """ get the code which implement the custom layer function
-    """
-    layer_type = node.layer_type
-    child_func = custom_layers[layer_type]['child_func']
-    if child_func is None:
-        return None, child_func
-    import inspect
-    return inspect.getsource(child_func), child_func
-
-
-def deal_weights(node, data=None):
-    """ deal the weights of the custom layer
-    """
-    layer_type = node.layer_type
-    weights_func = custom_layers[layer_type]['weights']
-    name = node.layer_name
-    return weights_func(name, data)

x2paddle/op_mapper/static/onnx2paddle/opset9/custom_layer/register.py

-#   Copyright (c) 2019  PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" this module provides 'register' for registering customized layers
-"""
-
-g_custom_layers = {}
-
-
-def register(kind, shape, layer, child_func, weights):
-    """ register a custom layer or a list of custom layers
-
-    Args:
-        @kind (str or list): type name of the layer
-        @shape (function): a function to generate the shape of layer's output
-        @layer (function): a function to generate the paddle code of layer
-        @weights (function): a function to deal with weights data
-
-    Returns:
-        None
-    """
-    assert type(shape).__name__ == 'function', 'shape should be a function'
-    assert type(layer).__name__ == 'function', 'layer should be a function'
-
-    if type(kind) is str:
-        kind = [kind]
-    else:
-        assert type(
-            kind) is list, 'invalid param "kind" for register, not a list or str'
-
-    for k in kind:
-        assert type(
-            k) is str, 'invalid param "kind" for register, not a list of str'
-        assert k not in g_custom_layers, 'this type[%s] has already been registered' % (
-            k)
-        g_custom_layers[k] = {
-            'shape': shape,
-            'layer': layer,
-            'child_func': child_func,
-            'weights': weights
-        }
-
-
-def get_registered_layers():
-    return g_custom_layers

x2paddle/optimizer/onnx_optimizer.py

-#   Copyright (c) 2019  PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# TODO useless node remove
-
-
-class ONNXOptimizer(object):
-    def __init__(self, op_mapper):
-        self.op_mapper = op_mapper
-        self.graph = op_mapper.graph
-
-    def delete_redundance_code(self):
-        for node_name in self.graph.topo_sort:
-            if node_name in self.op_mapper.omit_nodes:
-                node = self.graph.get_node(node_name)
-                omit_freq = self.op_mapper.omit_nodes.count(node_name)
-                if len(node.outputs) <= omit_freq:
-                    node.fluid_code.clear()