Skip to content

X
X2Paddle

PaddlePaddle / X2Paddle
大约 1 年 前同步成功

通知 328
Star 698
Fork 167
X
X2Paddle
提交 262229f5 编写于 作者: S SunAhong1993
浏览文件
操作

modify onnx static

上级 9e19ff2b
展开全部 隐藏空白更改
内联 并排
Showing with 888 addition and 1080 deletion
x2paddle/convert.py
浏览文件 @ 262229f5
......@@ -185,16 +185,8 @@ def onnx2paddle(model_path, save_dir, paddle_type, params_merge=False):
from x2paddle.op_mapper.static.onnx2paddle.onnx_op_mapper import ONNXOpMapper
model = ONNXDecoder(model_path)
mapper = ONNXOpMapper(model)
if paddle_type == "dygraph":
mapper.paddle_graph.build()
mapper.paddle_graph.gen_model(save_dir)
else:
from x2paddle.optimizer.onnx_optimizer import ONNXOptimizer
print("Model optimizing ...")
optimizer = ONNXOptimizer(mapper)
optimizer.delete_redundance_code()
print("Model optimized.")
mapper.save_inference_model(save_dir, params_merge)
mapper.paddle_graph.build()
mapper.paddle_graph.gen_model(save_dir)
def pytorch2paddle(module, save_dir, jit_type="trace", input_examples=None):
......
x2paddle/op_mapper/dygraph/onnx2paddle/opset9/opset.py
浏览文件 @ 262229f5
......@@ -534,7 +534,7 @@ class OpSet9():
'bias_attr': string(val_b.name)
}
dim = len(val_x.out_shapes[0])
if dim == 2 or dim == 3:
if dim == 3:
paddle_op = "paddle.nn.InstanceNorm1D"
elif dim == 4:
paddle_op = "paddle.nn.InstanceNorm2D"
......@@ -1539,7 +1539,6 @@ class OpSet9():
layer_outputs = [op_name, output_name]
val_x = self.graph.get_input_node(node, idx=0, copy=True)
val_w = self.graph.get_input_node(node, idx=1, copy=True)
val_y = self.graph.get_node(node.layer.output[0], copy=True)
has_bias = len(node.layer.input) == 3
if has_bias:
val_b = self.graph.get_input_node(node, idx=2, copy=True)
......@@ -1589,6 +1588,9 @@ class OpSet9():
@print_mapping_info
def ConvTranspose(self, node):
op_name = name_generator("conv", self.nn_name2id)
output_name = node.name
layer_outputs = [op_name, output_name]
val_x = self.graph.get_input_node(node, idx=0, copy=True)
val_w = self.graph.get_input_node(node, idx=1, copy=True)
val_b = None
......@@ -1602,7 +1604,7 @@ class OpSet9():
assert 2 <= convnd <= 3, 'only Conv2DTranspose and Conv3DTranspose supported'
num_in_channels = val_w.out_shapes[0][0]
num_out_channels = val_w.out_shapes[0][1]
paddle_op = 'paddle.nn.functional.conv{}d_transpose'.format(convnd)
paddle_op = 'paddle.nn.Conv{}DTranspose'.format(convnd)
num_groups = node.get_attr('group', 1)
strides = node.get_attr('strides', [1] * convnd)
......@@ -1620,37 +1622,37 @@ class OpSet9():
output_size[1] = (val_x.out_shapes[0][3] - 1
) * strides[1] - 2 * paddings[1] + dilations[1] * (
kernel_shape[1] - 1) + 1 + out_padding[1]
# layer_attrs = {
# 'in_channels': num_in_channels,
# 'out_channels': num_out_channels,
# 'output_size': output_size or None,
# 'kernel_size': kernel_shape,
# 'padding': paddings,
# 'stride': strides,
# 'dilation': dilations,
# 'groups': num_groups,
# 'weight_attr': string(val_w.name),
# 'bias_attr': None if val_b is None else string(val_b.name),
# }
# self.paddle_graph.add_layer(
# paddle_op,
# inputs={"x": val_x.name},
# outputs=layer_outputs,
# **layer_attrs)
inputs_dict = {'x': val_x if isinstance(val_x, str) else val_x.name,
"weight": val_w.name}
layer_attrs = {
"stride": strides,
"dilation": dilations,
"padding": paddings,
"groups": num_groups,
"output_size": node.out_shapes[0][2:]}
if val_b is not None:
inputs_dict["bias"] = val_b.name
else:
layer_attrs["bias"] = None
'in_channels': num_in_channels,
'out_channels': num_out_channels,
'output_size': output_size or None,
'kernel_size': kernel_shape,
'padding': paddings,
'stride': strides,
'dilation': dilations,
'groups': num_groups,
'weight_attr': string(val_w.name),
'bias_attr': None if val_b is None else string(val_b.name),
}
self.paddle_graph.add_layer(
kernel="paddle.nn.functional.conv2d_transpose",
inputs=inputs_dict,
outputs=[node.name],
paddle_op,
inputs={"x": val_x.name},
outputs=layer_outputs,
**layer_attrs)
# inputs_dict = {'x': val_x if isinstance(val_x, str) else val_x.name,
# "weight": val_w.name}
# layer_attrs = {
# "stride": strides,
# "dilation": dilations,
# "padding": paddings,
# "groups": num_groups,
# "output_size": node.out_shapes[0][2:]}
# if val_b is not None:
# inputs_dict["bias"] = val_b.name
# else:
# layer_attrs["bias"] = None
# self.paddle_graph.add_layer(
# kernel="paddle.nn.functional.conv2d_transpose",
# inputs=inputs_dict,
# outputs=[node.name],
# **layer_attrs)
x2paddle/op_mapper/static/onnx2paddle/onnx_op_mapper.py
浏览文件 @ 262229f5
......@@ -12,9 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from x2paddle.op_mapper.static.onnx2paddle.opset9 import OpSet9, custom_layers
import sys
from x2paddle.op_mapper.static.onnx2paddle.opset9 import OpSet9
from x2paddle.core.op_mapper import OpMapper
from x2paddle.decoder.onnx_decoder import ONNXGraph, ONNXGraphNode, ONNXGraphDataNode
from x2paddle.decoder.onnx_decoder import ONNXGraphNode
from x2paddle.core.program import PaddleGraph
class ONNXOpMapper(OpMapper):
......@@ -23,33 +25,36 @@ class ONNXOpMapper(OpMapper):
self.support_op_sets = [9, ]
self.default_op_set = 9
self.graph = decoder.graph
self.paddle_graph = PaddleGraph(parent_layer=None, graph_type="static", source_type="onnx")
self.paddle_graph.outputs = self.graph.output_nodes
self.opset = self.create_opset(decoder)
if not self.op_checker():
raise Exception("Model are not supported yet.")
#mapping op
raise Exception("Model is not supported yet.")
print("Total nodes: {}".format(
sum([
isinstance(node, ONNXGraphNode)
for name, node in self.graph.node_map.items()
])))
print("Nodes converting ...")
for node_name in self.graph.topo_sort:
for i, node_name in enumerate(self.graph.topo_sort):
sys.stderr.write("\rConverting node {} ... ".format(i + 1))
node = self.graph.get_node(node_name)
op = node.layer_type
if hasattr(self.opset, op):
func = getattr(self.opset, op)
func(node)
elif op in self.opset.default_op_mapping:
elif op in self.opset.directly_map_ops:
self.opset.directly_map(node)
elif op in custom_layers:
self.opset.deal_custom_layer(node)
elif op in self.opset.elementwise_ops:
self.opset.elementwise_map(node)
print("Nodes converted.")
self.weights = self.opset.weights
self.omit_nodes = self.opset.omit_nodes
self.used_custom_layers = self.opset.used_custom_layers
print("\nNodes converted.")
self.paddle_graph.set_name(self.graph.graph_name)
self.paddle_graph.set_parameters(self.opset.params)
self.paddle_graph.set_inputs_info(self.opset.inputs_info)
self.paddle_graph.inputs = self.graph.input_nodes
self.paddle_graph.outputs = self.graph.output_nodes
def op_checker(self):
unsupported_ops = set()
......@@ -57,17 +62,17 @@ class ONNXOpMapper(OpMapper):
node = self.graph.get_node(node_name)
op = node.layer_type
if not hasattr(self.opset, op) and \
op not in self.opset.default_op_mapping and \
op not in custom_layers and \
op not in self.opset.directly_map_ops and \
op not in self.opset.elementwise_ops:
unsupported_ops.add(op)
if len(unsupported_ops) == 0:
return True
else:
print("There are {} ops not supported yet, list as below".format(
len(unsupported_ops)))
if len(unsupported_ops) > 0:
print("\n========= {} OPs are not supported yet ===========".format(
len(unsupported_ops)))
for op in unsupported_ops:
print(op)
print("========== {} ============".format(op))
return False
def create_opset(self, decoder):
......@@ -88,4 +93,4 @@ class ONNXOpMapper(OpMapper):
'Now, onnx2paddle support convert onnx model opset_verison {},'
'opset_verison of your onnx model is {}, automatically treated as op_set: {}.'
.format(self.support_op_sets, decoder.op_set, run_op_set))
return eval(opset)(decoder)
return eval(opset)(decoder, self.paddle_graph)
x2paddle/op_mapper/static/onnx2paddle/opset9/custom_layer/__init__.py 已删除 100644 → 0
浏览文件 @ 9e19ff2b
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .register import get_registered_layers
custom_layers = get_registered_layers()
def set_args(f, params):
""" set args for function 'f' using the parameters in node.layer.param
Args:
f (function): a python function object
params (object): a object contains attributes needed by f's arguments
Returns:
arg_names (list): a list of argument names
kwargs (dict): a dict contains needed arguments
"""
argc = f.__code__.co_argcount
arg_list = f.__code__.co_varnames[0:argc]
kwargs = {}
for arg_name in arg_list:
if hasattr(params, arg_name) and params is not None:
kwargs[arg_name] = getattr(params, arg_name)
return arg_list, kwargs
def has_layer(layer_type):
""" test whether this layer exists in custom layer
"""
return layer_type in custom_layers
def get_params(layer, layer_type):
import re
if layer_type.lower() == "deconvolution" or layer_type.lower(
) == "convolutiondepthwise":
param_name = '_'.join(('convolution', 'param'))
elif layer_type.lower() == "normalize":
param_name = '_'.join(('norm', 'param'))
elif len(layer_type) - len(re.sub("[A-Z]", "", layer_type)) >= 2:
s = ''
tmp_name = ''
for i, ch in enumerate(layer_type):
if i == 0:
s += ch.lower()
continue
elif ch.isupper() and layer_type[i - 1].islower():
tmp_name += (s + '_')
s = ''
s += ch.lower()
tmp_name += s
param_name = '_'.join((tmp_name, 'param'))
else:
param_name = '_'.join((layer_type.lower(), 'param'))
return getattr(layer, param_name, None)
def compute_output_shape(node):
""" compute the output shape of custom layer
"""
layer_type = node.layer_type
assert layer_type in custom_layers, "layer[%s] not exist in custom layers" % (
layer_type)
shape_func = custom_layers[layer_type]['shape']
layer = node.layer
params = get_params(layer, layer_type)
arg_names, kwargs = set_args(shape_func, params)
input_shape = node.input_shape
return shape_func(input_shape, **kwargs)
def make_custom_layer(node):
""" get the code which implement the custom layer function
"""
layer_type = node.layer_type
assert layer_type in custom_layers, "layer[%s] not exist in custom layers" % (
layer_type)
layer_func = custom_layers[layer_type]['layer']
import inspect
return inspect.getsource(layer_func), layer_func
def make_custom_child_func(node):
""" get the code which implement the custom layer function
"""
layer_type = node.layer_type
child_func = custom_layers[layer_type]['child_func']
if child_func is None:
return None, child_func
import inspect
return inspect.getsource(child_func), child_func
def deal_weights(node, data=None):
""" deal the weights of the custom layer
"""
layer_type = node.layer_type
weights_func = custom_layers[layer_type]['weights']
name = node.layer_name
return weights_func(name, data)
x2paddle/op_mapper/static/onnx2paddle/opset9/custom_layer/register.py 已删除 100644 → 0
浏览文件 @ 9e19ff2b
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" this module provides 'register' for registering customized layers
"""
g_custom_layers = {}
def register(kind, shape, layer, child_func, weights):
""" register a custom layer or a list of custom layers
Args:
@kind (str or list): type name of the layer
@shape (function): a function to generate the shape of layer's output
@layer (function): a function to generate the paddle code of layer
@weights (function): a function to deal with weights data
Returns:
None
"""
assert type(shape).__name__ == 'function', 'shape should be a function'
assert type(layer).__name__ == 'function', 'layer should be a function'
if type(kind) is str:
kind = [kind]
else:
assert type(
kind) is list, 'invalid param "kind" for register, not a list or str'
for k in kind:
assert type(
k) is str, 'invalid param "kind" for register, not a list of str'
assert k not in g_custom_layers, 'this type[%s] has already been registered' % (
k)
g_custom_layers[k] = {
'shape': shape,
'layer': layer,
'child_func': child_func,
'weights': weights
}
def get_registered_layers():
return g_custom_layers
x2paddle/optimizer/onnx_optimizer.py 已删除 100644 → 0
浏览文件 @ 9e19ff2b
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# TODO useless node remove
class ONNXOptimizer(object):
def __init__(self, op_mapper):
self.op_mapper = op_mapper
self.graph = op_mapper.graph
def delete_redundance_code(self):
for node_name in self.graph.topo_sort:
if node_name in self.op_mapper.omit_nodes:
node = self.graph.get_node(node_name)
omit_freq = self.op_mapper.omit_nodes.count(node_name)
if len(node.outputs) <= omit_freq:
node.fluid_code.clear()
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或