Skip to content

X
X2Paddle

PaddlePaddle / X2Paddle
大约 1 年 前同步成功

通知 328
Star 698
Fork 167
X
X2Paddle
提交 2cfdb2c8 编写于 作者: S SunAhong1993
浏览文件
操作

rm x2paddle

上级 c66440ea
展开全部 隐藏空白更改
内联 并排
Showing with 0 addition and 26016 deletion
x2paddle/x2paddle/__init__.py 已删除 100644 → 0
浏览文件 @ c66440ea
__version__ = "0.8.4"
from .core.program import PaddleProgram
program = PaddleProgram()
name_counter = dict()
def gen_name(op_name, var_name):
name = "{}.{}".format(op_name, var_name)
if name not in name_counter:
name_counter[name] = 0
else:
name_counter[name] += 1
name = name + "." + str(name_counter[name])
return name
x2paddle/x2paddle/convert.py 已删除 100644 → 0
浏览文件 @ c66440ea
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from six import text_type as _text_type
import argparse
import sys
def arg_parser():
parser = argparse.ArgumentParser()
parser.add_argument(
"--model",
"-m",
type=_text_type,
default=None,
help="define model file path for tensorflow or onnx")
parser.add_argument(
"--prototxt",
"-p",
type=_text_type,
default=None,
help="prototxt file of caffe model")
parser.add_argument(
"--weight",
"-w",
type=_text_type,
default=None,
help="weight file of caffe model")
parser.add_argument(
"--save_dir",
"-s",
type=_text_type,
default=None,
help="path to save translated model")
parser.add_argument(
"--framework",
"-f",
type=_text_type,
default=None,
help="define which deeplearning framework(tensorflow/caffe/onnx/paddle2onnx)"
)
parser.add_argument(
"--caffe_proto",
"-c",
type=_text_type,
default=None,
help="optional: the .py file compiled by caffe proto file of caffe model"
)
parser.add_argument(
"--version",
"-v",
action="store_true",
default=False,
help="get version of x2paddle")
parser.add_argument(
"--without_data_format_optimization",
"-wo",
type=_text_type,
default="True",
help="tf model conversion without data format optimization")
parser.add_argument(
"--define_input_shape",
"-d",
action="store_true",
default=False,
help="define input shape for tf model")
parser.add_argument(
"--onnx_opset",
"-oo",
type=int,
default=10,
help="when paddle2onnx set onnx opset version to export")
parser.add_argument(
"--params_merge",
"-pm",
action="store_true",
default=False,
help="define whether merge the params")
return parser
def tf2paddle(model_path,
save_dir,
without_data_format_optimization,
define_input_shape=False,
params_merge=False):
# check tensorflow installation and version
try:
import os
os.environ["TF_CPP_MIN_LOG_LEVEL"] = '3'
import tensorflow as tf
version = tf.__version__
if version >= '2.0.0' or version < '1.0.0':
print(
"[ERROR] 1.0.0<=tensorflow<2.0.0 is required, and v1.14.0 is recommended"
)
return
except:
print(
"[ERROR] Tensorflow is not installed, use \"pip install tensorflow\"."
)
return
from x2paddle.decoder.tf_decoder import TFDecoder
from x2paddle.op_mapper.tf_op_mapper import TFOpMapper
from x2paddle.op_mapper.tf_op_mapper_nhwc import TFOpMapperNHWC
from x2paddle.optimizer.tf_optimizer import TFOptimizer
print("Now translating model from tensorflow to paddle.")
model = TFDecoder(model_path, define_input_shape=define_input_shape)
if not without_data_format_optimization:
mapper = TFOpMapper(model)
optimizer = TFOptimizer(mapper)
# neccesary optimization
optimizer.delete_redundance_code()
# optimizer below is experimental
optimizer.optimize_elementwise_op()
optimizer.merge_activation()
optimizer.merge_bias()
optimizer.optimize_sub_graph()
# optimizer.merge_batch_norm()
# optimizer.merge_prelu()
else:
mapper = TFOpMapperNHWC(model)
optimizer = TFOptimizer(mapper)
optimizer.delete_redundance_code()
optimizer.strip_graph()
optimizer.merge_activation()
optimizer.merge_bias()
optimizer.make_nchw_input_output()
optimizer.remove_transpose()
mapper.save_inference_model(save_dir, params_merge)
def caffe2paddle(proto, weight, save_dir, caffe_proto, params_merge=False):
from x2paddle.decoder.caffe_decoder import CaffeDecoder
from x2paddle.op_mapper.caffe_op_mapper import CaffeOpMapper
from x2paddle.optimizer.caffe_optimizer import CaffeOptimizer
import google.protobuf as gpb
ver_part = gpb.__version__.split('.')
version_satisfy = False
if (int(ver_part[0]) == 3 and int(ver_part[1]) >= 6) \
or (int(ver_part[0]) > 3):
version_satisfy = True
assert version_satisfy, '[ERROR] google.protobuf >= 3.6.0 is required'
print("Now translating model from caffe to paddle.")
model = CaffeDecoder(proto, weight, caffe_proto)
mapper = CaffeOpMapper(model)
optimizer = CaffeOptimizer(mapper)
optimizer.merge_bn_scale()
optimizer.merge_op_activation()
mapper.save_inference_model(save_dir, params_merge)
def onnx2paddle(model_path, save_dir, params_merge=False):
# check onnx installation and version
try:
import onnx
version = onnx.version.version
if version < '1.6.0':
print("[ERROR] onnx>=1.6.0 is required")
return
except:
print("[ERROR] onnx is not installed, use \"pip install onnx==1.6.0\".")
return
print("Now translating model from onnx to paddle.")
from x2paddle.op_mapper.onnx2paddle.onnx_op_mapper import ONNXOpMapper
from x2paddle.decoder.onnx_decoder import ONNXDecoder
from x2paddle.optimizer.onnx_optimizer import ONNXOptimizer
model = ONNXDecoder(model_path)
mapper = ONNXOpMapper(model)
print("Model optimizing ...")
optimizer = ONNXOptimizer(mapper)
print("Model optimized.")
print("Paddle model and code generating ...")
mapper.save_inference_model(save_dir, params_merge)
print("Paddle model and code generated.")
def paddle2onnx(model_path, save_dir, opset_version=10):
from x2paddle.decoder.paddle_decoder import PaddleDecoder
from x2paddle.op_mapper.paddle2onnx.paddle_op_mapper import PaddleOpMapper
import paddle.fluid as fluid
model = PaddleDecoder(model_path, '__model__', '__params__')
mapper = PaddleOpMapper()
mapper.convert(
model.program,
save_dir,
scope=fluid.global_scope(),
opset_version=opset_version)
def main():
if len(sys.argv) < 2:
print("Use \"x2paddle -h\" to print the help information")
print("For more information, please follow our github repo below:)")
print("\nGithub: https://github.com/PaddlePaddle/X2Paddle.git\n")
return
parser = arg_parser()
args = parser.parse_args()
if args.version:
import x2paddle
print("x2paddle-{} with python>=3.5, paddlepaddle>=1.6.0\n".format(
x2paddle.__version__))
return
assert args.framework is not None, "--framework is not defined(support tensorflow/caffe/onnx)"
assert args.save_dir is not None, "--save_dir is not defined"
try:
import paddle
v0, v1, v2 = paddle.__version__.split('.')
print("paddle.__version__ = {}".format(paddle.__version__))
if v0 == '0' and v1 == '0' and v2 == '0':
print("[WARNING] You are use develop version of paddlepaddle")
elif int(v0) != 1 or int(v1) < 6:
print("[ERROR] paddlepaddle>=1.6.0 is required")
return
except:
print(
"[ERROR] paddlepaddle not installed, use \"pip install paddlepaddle\""
)
if args.framework == "tensorflow":
assert args.model is not None, "--model should be defined while translating tensorflow model"
assert args.without_data_format_optimization in [
"True", "False"
], "--the param without_data_format_optimization should be defined True or False"
define_input_shape = False
params_merge = False
without_data_format_optimization = True if args.without_data_format_optimization == "True" else False
if args.define_input_shape:
define_input_shape = True
if args.params_merge:
params_merge = True
tf2paddle(args.model, args.save_dir, without_data_format_optimization,
define_input_shape, params_merge)
elif args.framework == "caffe":
assert args.prototxt is not None and args.weight is not None, "--prototxt and --weight should be defined while translating caffe model"
params_merge = False
if args.params_merge:
params_merge = True
caffe2paddle(args.prototxt, args.weight, args.save_dir,
args.caffe_proto, params_merge)
elif args.framework == "onnx":
assert args.model is not None, "--model should be defined while translating onnx model"
params_merge = False
if args.params_merge:
params_merge = True
onnx2paddle(args.model, args.save_dir, params_merge)
elif args.framework == "paddle2onnx":
assert args.model is not None, "--model should be defined while translating paddle model to onnx"
paddle2onnx(args.model, args.save_dir, opset_version=args.onnx_opset)
else:
raise Exception(
"--framework only support tensorflow/caffe/onnx/paddle2onnx now")
if __name__ == "__main__":
main()
x2paddle/x2paddle/core/fluid_code.py 已删除 100644 → 0
浏览文件 @ c66440ea
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from x2paddle.core.graph import GraphNode
from x2paddle.core.util import *
import collections
import six
class Layer(object):
def __init__(self):
self.op = None
self.param_attr = dict()
self.inputs = dict()
self.output = None
self.is_custom_layer = False
self.use_fluid = False
def get_code(self):
layer_code = ""
if self.output is not None:
if isinstance(self.output, six.string_types):
layer_code = self.output + " = "
else:
layer_code = self.output.layer_name + " = "
if self.is_custom_layer:
layer_code = layer_code + self.op + "("
elif self.op == "=":
layer_code = layer_code
elif self.use_fluid:
layer_code = layer_code + "fluid." + self.op + "("
else:
layer_code = layer_code + "fluid.layers." + self.op + "("
if isinstance(self.inputs, list):
in_list = "["
for input in self.inputs:
if isinstance(input, GraphNode):
if hasattr(input, "index"):
in_list += (
input.layer_name + "[{}]".format(input.index) + ", "
)
else:
in_list += (input.layer_name + ", ")
elif isinstance(input, six.string_types):
in_list += (input + ", ")
else:
raise Exception(
"Element of inputs should GraphNode or String")
in_list = in_list.strip(", ") + "], "
layer_code += in_list
elif isinstance(self.inputs, dict):
inputs = collections.OrderedDict(self.inputs)
for key, input in inputs.items():
if isinstance(input, GraphNode):
if hasattr(input, "index"):
layer_code = layer_code + key + "={}, ".format(
input.layer_name + "[{}]".format(input.index))
else:
layer_code = layer_code + key + "={}, ".format(
input.layer_name)
else:
layer_code = layer_code + key + "={}, ".format(input)
elif isinstance(self.inputs, GraphNode):
if hasattr(self.inputs, "index"):
layer_code += (
self.inputs.layer_name + "[{}]".format(self.inputs.index))
else:
layer_code += (self.inputs.layer_name)
if self.op != "=":
layer_code += ", "
elif isinstance(self.inputs, six.string_types):
layer_code += (self.inputs)
if self.op != "=":
layer_code += ", "
else:
raise Exception("Unknown type of inputs.")
param_attr = collections.OrderedDict(self.param_attr)
for key, value in param_attr.items():
if '\n' in str(value):
value = string(str(value).replace('\n', ','))
if str(key) == 'attr':
value = 'ParamAttr(' + str(value) + ')'
layer_code = layer_code + key + "={}, ".format(value)
layer_code = layer_code.strip(", ")
if self.op != "=":
layer_code += ")"
return layer_code
class FluidCode(object):
def __init__(self):
self.layers = list()
def add_layer(self,
op,
inputs,
output,
param_attr=None,
use_fluid=False,
is_custom_layer=False):
layer = Layer()
layer.op = op
layer.use_fluid = use_fluid
layer.is_custom_layer = is_custom_layer
if inputs is not None:
layer.inputs = inputs
layer.output = output
if param_attr is not None:
layer.param_attr = param_attr
self.layers.append(layer)
def add_note(self, note):
# note should be string
self.layers.append(note)
def clear(self):
self.layers = list()
def gen_codes(self):
codes = list()
for layer in self.layers:
if isinstance(layer, Layer):
codes.append(layer.get_code())
elif isinstance(layer, six.string_types):
codes.append(layer)
return codes
x2paddle/x2paddle/core/graph.py 已删除 100644 → 0
浏览文件 @ c66440ea
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import division
import collections
import copy as cp
class GraphNode(object):
def __init__(self, layer, layer_name=None):
self.inputs = list()
self.outputs = list()
self.layer = layer
assert layer_name is not None, "layer_name for GraphNode should not be None"
self.layer_name = layer_name
def __hash__(self):
return hash(self.layer.name)
def __eq__(self, other):
if self.layer.name == other.layer.name:
return True
return False
class Graph(object):
def __init__(self, model):
self.node_map = collections.OrderedDict()
self.input_nodes = list()
self.output_nodes = list()
self.topo_sort = list()
self.model = model
def build(self):
self.get_input_nodes()
self.get_output_nodes()
self.get_topo_sort()
def get_input_nodes(self):
for name, node in self.node_map.items():
name = name.replace('/', '_').replace('-', '_')
if len(node.inputs) == 0:
self.input_nodes.append(name)
def get_output_nodes(self):
for name, node in self.node_map.items():
name = name.replace('/', '_').replace('-', '_')
if len(node.outputs) == 0:
self.output_nodes.append(name)
def get_topo_sort(self):
num_inputs = dict()
for name, node in self.node_map.items():
num_inputs[name] = len(node.inputs)
self.topo_sort = self.input_nodes[:]
idx = 0
while idx < len(self.topo_sort):
current_node = self.node_map[self.topo_sort[idx]]
for node in current_node.outputs:
num_inputs[node] -= 1
if num_inputs[node] == 0:
self.topo_sort.append(node)
idx += 1
def get_node(self, name, copy=False):
if name not in self.node_map:
if name.split(':')[0] in self.node_map:
name_prefix, idx = name.split(':')
if copy:
node = cp.copy(self.node_map[name_prefix])
else:
node = self.node_map[name_prefix]
node.index = int(idx)
return node
else:
return None
else:
if copy:
node = cp.copy(self.node_map[name])
else:
node = self.node_map[name]
return node
def connect(self, src, dst):
if dst not in self.node_map:
raise Exception("node[{}] not in graph".format(dst))
self.node_map[dst].inputs.append(src)
self.node_map[src].outputs.append(dst)
x2paddle/x2paddle/core/op_mapper.py 已删除 100644 → 0
浏览文件 @ c66440ea
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paddle.fluid as fluid
from paddle.fluid.proto import framework_pb2
from x2paddle.core.util import *
import inspect
import os
def export_paddle_param(param, param_name, dir):
dtype_map = {
"int16": [framework_pb2.VarType.INT16, 'h'],
"int32": [framework_pb2.VarType.INT32, 'i'],
"int64": [framework_pb2.VarType.INT64, 'q'],
"float16": [framework_pb2.VarType.FP16, 'e'],
"float32": [framework_pb2.VarType.FP32, 'f'],
"float64": [framework_pb2.VarType.FP64, 'd'],
"bool": [framework_pb2.VarType.BOOL, None]
}
shape = param.shape
if str(param.dtype) in ['uint8', 'uint_8', 'bool']:
param = param.astype('int64')
if len(shape) == 0:
assert param.size == 1, "Unexpected situation happend!"
shape = [1]
assert str(
param.dtype) in dtype_map, "Unknown dtype {} of params: {}.".format(
str(param.dtype), param_name)
fp = open(os.path.join(dir, param_name), 'wb')
numpy.array([0], dtype='int32').tofile(fp)
numpy.array([0], dtype='int64').tofile(fp)
numpy.array([0], dtype='int32').tofile(fp)
tensor_desc = framework_pb2.VarType.TensorDesc()
tensor_desc.data_type = dtype_map[str(param.dtype)][0]
tensor_desc.dims.extend(shape)
desc_size = tensor_desc.ByteSize()
numpy.array([desc_size], dtype='int32').tofile(fp)
fp.write(tensor_desc.SerializeToString())
param.tofile(fp)
fp.close()
# This func will copy to generate code file
def run_net(param_dir="./"):
import os
inputs, outputs = x2paddle_net()
ops = fluid.default_main_program().global_block().ops
used_vars = list()
for op in ops:
used_vars += op.input_arg_names
tmp = list()
for input in inputs:
if isinstance(input, list):
for ipt in input:
if ipt.name not in used_vars:
continue
tmp.append(ipt)
else:
if input.name not in used_vars:
continue
tmp.append(input)
inputs = tmp
for i, out in enumerate(outputs):
if isinstance(out, list):
for out_part in out:
outputs.append(out_part)
del outputs[i]
exe = fluid.Executor(fluid.CPUPlace())
exe.run(fluid.default_startup_program())
def if_exist(var):
b = os.path.exists(os.path.join(param_dir, var.name))
return b
fluid.io.load_vars(
exe, param_dir, fluid.default_main_program(), predicate=if_exist)
class OpMapper(object):
def __init__(self):
self.paddle_codes = ""
self.tab = " "
self.net_code = list()
self.weights = dict()
self.inputs = list()
self.outputs = list()
def op_checker(self):
unsupported_ops = set()
for node_name in self.graph.topo_sort:
node = self.graph.get_node(node_name)
op = node.layer_type
if not hasattr(self, op):
unsupported_ops.add(op)
if len(unsupported_ops) == 0:
return True
else:
print("There are {} ops not supported yet, list as below".format(
len(unsupported_ops)))
for op in unsupported_ops:
print(op)
return False
def add_codes(self, codes, indent=0):
if isinstance(codes, list):
for code in codes:
self.paddle_codes += (
self.tab * indent + code.strip('\n') + '\n')
elif isinstance(codes, str):
self.paddle_codes += (self.tab * indent + codes.strip('\n') + '\n')
else:
raise Exception("Unknown type of codes")
def add_heads(self):
self.add_codes("from paddle.fluid.initializer import Constant")
self.add_codes("from paddle.fluid.param_attr import ParamAttr")
self.add_codes("import paddle.fluid as fluid")
self.add_codes("")
def save_inference_model(self, save_dir, params_merge):
self.save_python_model(save_dir)
import sys
import paddle.fluid as fluid
py_code_dir = os.path.join(save_dir, "model_with_code")
sys.path.append(py_code_dir)
import model
try:
inputs, outputs = model.x2paddle_net()
ops = fluid.default_main_program().global_block().ops
used_vars = list()
for op in ops:
used_vars += op.input_arg_names
for i, out in enumerate(outputs):
if isinstance(out, list):
for out_part in out:
outputs.append(out_part)
del outputs[i]
input_names = list()
for input in inputs:
if isinstance(input, list):
for ipt in input:
if ipt.name not in used_vars:
continue
input_names.append(ipt.name)
else:
if input.name not in used_vars:
continue
input_names.append(input.name)
exe = fluid.Executor(fluid.CPUPlace())
exe.run(fluid.default_startup_program())
def if_exist(var):
b = os.path.exists(
os.path.join(os.path.join(py_code_dir, var.name)))
return b
fluid.io.load_vars(
exe,
py_code_dir,
fluid.default_main_program(),
predicate=if_exist)
if params_merge:
fluid.io.save_inference_model(
dirname=os.path.join(save_dir, "inference_model"),
feeded_var_names=input_names,
target_vars=outputs,
executor=exe,
params_filename="__params__")
else:
fluid.io.save_inference_model(
dirname=os.path.join(save_dir, "inference_model"),
feeded_var_names=input_names,
target_vars=outputs,
executor=exe,
params_filename=None)
except:
raise Exception(
"Paddle code was saved in {}/model.py, but seems there's wrong exist, please check model.py manually."
.format(py_code_dir))
def save_python_model(self, save_dir):
if not os.path.exists(save_dir):
os.makedirs(save_dir)
py_code_dir = os.path.join(save_dir, "model_with_code")
if not os.path.exists(py_code_dir):
os.makedirs(py_code_dir)
for name, param in self.weights.items():
export_paddle_param(param, name, py_code_dir)
self.add_heads()
if hasattr(self, "used_custom_layers"):
for _, layer_code in self.used_custom_layers.items():
self.add_codes(layer_code, 0)
self.add_codes("", 0)
self.add_codes("\ndef x2paddle_net():", 0)
for i in range(len(self.graph.topo_sort)):
node_name = self.graph.topo_sort[i]
node = self.graph.get_node(node_name)
if node is None:
continue
if len(node.fluid_code.layers) == 0:
continue
self.add_codes(node.fluid_code.gen_codes(), 1)
self.add_codes("", 0)
input_str = "["
for name in self.graph.input_nodes:
input_str += (name + ", ")
input_str = input_str.strip(", ") + "]"
output_str = "["
for name in self.graph.output_nodes:
output_str += (name + ", ")
output_str = output_str.strip(", ") + "]"
return_code = "return {}, {}".format(input_str, output_str)
self.add_codes(return_code, 1)
self.add_codes("", 0)
self.add_codes(inspect.getsourcelines(run_net)[0])
fp = open(os.path.join(py_code_dir, "model.py"), 'w')
fp.write(self.paddle_codes)
fp.close()
x2paddle/x2paddle/core/program.py 已删除 100644 → 0
浏览文件 @ c66440ea
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import division
import collections
import os
class PaddleLayer(object):
def __init__(self, kernel, inputs, outputs, **kwargs):
assert isinstance(
inputs,
dict), "parameter 'inputs' for PaddleLayer should be type of dict"
assert isinstance(
outputs,
list), "parameter, 'outputs' for PaddleLayer should be type of list"
self.kernel = kernel
self.inputs = inputs
self.outputs = outputs
self.attrs = kwargs
class PaddleProgram(object):
def __init__(self):
self.layers = list()
self.edges_out = dict()
self.edges_in = dict()
self.inputs = list()
self.outputs = list()
self.parameters = dict()
def add_layer(self, kernel, inputs, outputs, **kwargs):
layer = PaddleLayer(kernel, inputs, outputs, **kwargs)
self.layers.append(layer)
def build(self):
outputs = dict()
for i in range(len(self.layers)):
layer = self.layers[i]
for output in layer.outputs:
outputs[output] = i
for k, v in layer.inputs.items():
assert v in outputs, "Couldn't find {} in previous layers, the layers should be make by topological sort".format(
v)
in_layer_index = outputs[v]
if in_layer_index not in self.edges_out:
self.edges_out[in_layer_index] = list()
self.edges_out[in_layer_index].append(i)
if i not in self.edges_in:
self.edges_in[i] = list()
self.edges_in[i].append(in_layer_index)
def get_layer_outputs(self, i):
return self.edges_out[i]
def get_layer_inputs(self, i):
return self.edges_in[i]
def gen_code(self, code_dir):
def write_code(f, code_list, indent=0):
indent_blank = " " * indent
for code_line in code_list:
if code_line.strip() == "":
f.write('\n')
else:
f.write(indent_blank + code_line + '\n')
f = open(os.path.join(code_dir, 'model.py'), 'w')
write_code(
f, [
"from paddle.fluid.initializer import Constant",
"from paddle.fluid.param_attr import ParamAttr",
"import paddle.fluid as fluid"
"", "def x2paddle_net():"
],
indent=0)
for i, layer in enumerate(self.layers):
if self.edges_in.get(i, 0) == 0 and self.edges_out.get(i, 0) == 0:
continue
line = ""
if len(layer.outputs) == 1:
line = layer.outputs[0]
else:
for output in layer.outputs:
line += "{}, ".format(output)
line = line.strip(", ")
line += " = {}(".format(layer.kernel)
for k, v in layer.inputs.items():
line += "{}={}, ".format(k, v)
for k, v in layer.attrs.items():
line += "{}={}, ".format(k, v)
line = line.strip(", ")
line += ")"
write_code(f, [line], indent=1)
f.close()
def gen_parameters(self, code_dir):
pass
def gen_inference_model(self, model_dir):
pass
x2paddle/x2paddle/core/util.py 已删除 100644 → 0
浏览文件 @ c66440ea
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy
import math
import os
def string(param):
return "\'{}\'".format(param)
x2paddle/x2paddle/decoder/caffe.proto 已删除 100644 → 0
浏览文件 @ c66440ea
此差异已折叠。
x2paddle/x2paddle/decoder/caffe_decoder.py 已删除 100644 → 0
浏览文件 @ c66440ea
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
from google.protobuf import text_format
import numpy as np
from x2paddle.core.graph import GraphNode, Graph
from x2paddle.core.fluid_code import FluidCode
from x2paddle.op_mapper import caffe_shape
class CaffeResolver(object):
def __init__(self, caffe_proto):
self.caffe_proto = caffe_proto
self.import_caffe()
def import_caffepb(self):
if self.caffe_proto is None:
from x2paddle.decoder import caffe_pb2
out = caffe_pb2
else:
if not os.path.isfile(self.caffe_proto):
raise Exception(
"The .py file compiled by caffe.proto is not exist.")
(filepath,
tempfilename) = os.path.split(os.path.abspath(self.caffe_proto))
(filename, extension) = os.path.splitext(tempfilename)
sys.path.append(filepath)
out = __import__(filename)
return out
def import_caffe(self):
self.caffepb = self.import_caffepb()
self.NetParameter = self.caffepb.NetParameter
class CaffeGraphNode(GraphNode):
def __init__(self, layer, type_str, layer_name=None):
if layer_name is None:
super(CaffeGraphNode, self).__init__(
layer, layer.name.replace('/', '_').replace('-', '_'))
else:
super(CaffeGraphNode, self).__init__(
layer, layer_name.replace('/', '_').replace('-', '_'))
self.layer_type = type_str
self.fluid_code = FluidCode()
self.data = None
def set_params(self, params):
self.data = params
class CaffeGraph(Graph):
def __init__(self, model, params, caffe_pb):
self.params = params
self.caffe_pb = caffe_pb
super(CaffeGraph, self).__init__(model)
def filter_layers(self, layers):
'''Filter out layers based on the current phase.'''
phase_map = {0: 'train', 1: 'test'}
filtered_layer_names = set()
filtered_layers = []
for layer in layers:
if hasattr(layer, 'input'):
continue
type_str = self.get_layer_type(layer)
phase = 'test'
if len(layer.include):
phase = phase_map[layer.include[0].phase]
if len(layer.exclude):
phase = phase_map[1 - layer.include[0].phase]
exclude = (phase != 'test')
# Dropout layers appear in a fair number of Caffe
# test-time networks. These are just ignored. We'll
# filter them out here.
if (not exclude) and (phase == 'test'):
exclude = (type_str == 'Dropout')
if layer.type == 'Dropout':
drop_layer_top = layer.top[0]
drop_layer_bottom = layer.bottom[0]
if drop_layer_top != drop_layer_bottom:
for next_layer in layers:
for next_layer_bottom_idx, next_layer_bottom in enumerate(
next_layer.bottom):
if drop_layer_top == next_layer_bottom:
next_layer.bottom.remove(drop_layer_top)
next_layer.bottom.insert(
next_layer_bottom_idx,
drop_layer_bottom)
if not exclude:
filtered_layers.append(layer)
# Guard against dupes.
assert layer.name not in filtered_layer_names
filtered_layer_names.add(layer.name)
else:
print('The filter layer:' + layer.name)
return filtered_layers
def generate_input_layer(self, dims, index):
dim_str = ''
for dim in dims:
dim_str += 'dim: {}\n'.format(str(dim))
input_str = 'layer {\n'
input_str += 'name: \"{}\"\n '.format(str(self.model.input[index]))
input_str += 'type: "Input"\n'
input_str += 'top: \"{}\"\n'.format(str(self.model.input[index]))
input_str += 'input_param {\n'
input_str += 'shape {\n'
input_str += dim_str
input_str += '}}}'
input_str = str.encode(input_str)
net = self.caffe_pb.NetParameter()
text_format.Merge(input_str, net)
return net.layers or net.layer
def input2layers(self, input_layers=[]):
inputs_num = len(self.model.input)
if inputs_num != 0:
input_dims_num = len(self.model.input_dim)
if input_dims_num != 0:
if input_dims_num > 0 and input_dims_num != inputs_num * 4:
raise Error('invalid input_dim[%d] param in prototxt' %
(input_dims_num))
for i in range(inputs_num):
dims = self.model.input_dim[i * 4:(i + 1) * 4]
l = self.generate_input_layer(dims, i)
input_layers.append(l[0])
else:
for i in range(inputs_num):
dims = self.model.input_shape[i].dim[0:4]
l = self.generate_input_layer(dims, i)
input_layers.append(l[0])
def transform_input_layers(self, layers, input_layers=[]):
for layer in layers:
if hasattr(layer, 'input'):
input_dims_num = len(layers.input_dim)
if input_dims_num > 0 and input_dims_num != 4:
raise Error('invalid input_dim[%d] param in prototxt' %
(input_dims_num))
dims = self.model.input_dim[0:4]
l = self.generate_input_layer(dims, i)
input_layers.append(l[0])
def get_layer_type(self, layer):
if isinstance(layer.type, int):
enum_values = self.caffe_pb._V1LAYERPARAMETER_LAYERTYPE.values
vals = [val for val in enum_values if val.number == layer.type]
part = vals[0].name.split('_')
part = [s.capitalize() for s in part]
type_str = ''
type_str = type_str.join(part)
if 'relu' in type_str.lower():
type_str = type_str.replace('elu', 'eLU')
elif type_str.lower() == 'lrn':
type_str = 'LRN'
return type_str
else:
return layer.type
def build(self):
layers = self.model.layers or self.model.layer
layers = self.filter_layers(layers)
input_layers = []
self.input2layers(input_layers)
self.transform_input_layers(layers, input_layers)
layers = input_layers + layers
for layer in layers:
if hasattr(layer, 'name'):
name = getattr(layer, 'name')
setattr(layer, 'name', name.replace('/', '_').replace('-', '_'))
for i, name in enumerate(layer.bottom):
layer.bottom[i] = name.replace('/', '_').replace('-', '_')
for i, name in enumerate(layer.top):
layer.top[i] = name.replace('/', '_').replace('-', '_')
top_layer = {}
for layer in layers:
if hasattr(layer, 'input'):
continue
type_str = self.get_layer_type(layer)
self.node_map[layer.name] = CaffeGraphNode(layer, type_str)
for in_name in layer.bottom:
if in_name in top_layer:
self.connect(top_layer[in_name][-1], layer.name)
else:
raise Exception(
'input[{}] of node[{}] does not exist in node_map'.
format(in_name, layer.name))
for out_name in layer.top:
if out_name not in top_layer:
top_layer[out_name] = [layer.name]
else:
top_layer[out_name].append(layer.name)
for layer_name, data in self.params:
if layer_name in self.node_map:
node = self.node_map[layer_name]
node.set_params(data)
else:
print('Ignoring parameters for non-existent layer: %s' % \
layer_name)
super(CaffeGraph, self).build()
def get_bottom_node(self, node, idx=0, copy=False):
input_node_name = node.inputs[idx]
assert input_node_name in self.node_map, 'The {} isn\'t a valid node'.format(
name)
input_node = self.node_map[input_node_name]
if len(input_node.layer.top) > 1:
need_idx = list(input_node.layer.top).index(node.layer.bottom[idx])
name = input_node_name + ':' + str(need_idx)
else:
name = input_node_name
return self.get_node(name, copy=copy)
class CaffeDecoder(object):
def __init__(self, proto_path, model_path, caffe_proto):
self.proto_path = proto_path
self.model_path = model_path
self.resolver = CaffeResolver(caffe_proto=caffe_proto)
self.net = self.resolver.NetParameter()
with open(proto_path, 'rb') as proto_file:
proto_str = proto_file.read()
text_format.Merge(proto_str, self.net)
self.load_using_pb()
self.caffe_graph = CaffeGraph(self.net, self.params,
self.resolver.caffepb)
self.caffe_graph.build()
def load_using_pb(self):
data = self.resolver.NetParameter()
data.MergeFromString(open(self.model_path, 'rb').read())
layers = data.layers or data.layer
for layer in layers:
setattr(layer, 'name',
layer.name.replace('/', '_').replace('-', '_'))
pair = lambda layer: (layer.name, self.normalize_pb_data(layer))
self.params = [pair(layer) for layer in layers if layer.blobs]
def normalize_pb_data(self, layer):
transformed = []
for blob in layer.blobs:
if len(blob.shape.dim):
dims = blob.shape.dim
if layer.type == 'PReLU':
c_o, c_i, h, w = map(int, [1] + \
list(dims) + [1]* (3 - len(dims)))
elif layer.type == 'Normalize' and len(dims) == 4:
data = np.asarray(list(blob.data), dtype=np.float32)
transformed.append(data)
continue
else:
c_o, c_i, h, w = map(int,
[1] * (4 - len(dims)) + list(dims))
else:
c_o = blob.num
c_i = blob.channels
h = blob.height
w = blob.width
data = np.asarray(
list(blob.data), dtype=np.float32).reshape(c_o, c_i, h, w)
transformed.append(data)
return transformed
x2paddle/x2paddle/decoder/caffe_pb2.py 已删除 100644 → 0
浏览文件 @ c66440ea
此差异已折叠。
x2paddle/x2paddle/decoder/onnx_decoder.py 已删除 100644 → 0
浏览文件 @ c66440ea
此差异已折叠。
x2paddle/x2paddle/decoder/paddle_decoder.py 已删除 100644 → 0
浏览文件 @ c66440ea
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paddle.fluid as fluid
class PaddleDecoder(object):
def __init__(self,
model_dir,
model_filename='__model__',
params_filename=None):
exe = fluid.Executor(fluid.CPUPlace())
[self.program, feed, fetchs] = fluid.io.load_inference_model(
model_dir,
exe,
model_filename=model_filename,
params_filename=params_filename)
x2paddle/x2paddle/decoder/tf_decoder.py 已删除 100644 → 0
浏览文件 @ c66440ea
此差异已折叠。
x2paddle/x2paddle/op_mapper/caffe_custom_layer/__init__.py 已删除 100644 → 0
浏览文件 @ c66440ea
from .register import get_registered_layers
#custom layer import begins
from . import roipooling
from . import priorbox
from . import permute
from . import detectionoutput
from . import normalize
from . import select
from . import shufflechannel
from . import convolutiondepthwise
from . import axpy
from . import upsample
from . import relu6
#custom layer import ends
custom_layers = get_registered_layers()
def set_args(f, params):
""" set args for function 'f' using the parameters in node.layer.param
Args:
f (function): a python function object
params (object): a object contains attributes needed by f's arguments
Returns:
arg_names (list): a list of argument names
kwargs (dict): a dict contains needed arguments
"""
argc = f.__code__.co_argcount
arg_list = f.__code__.co_varnames[0:argc]
kwargs = {}
for arg_name in arg_list:
if hasattr(params, arg_name) and params is not None:
kwargs[arg_name] = getattr(params, arg_name)
return arg_list, kwargs
def has_layer(layer_type):
""" test whether this layer exists in custom layer
"""
return layer_type in custom_layers
def get_params(layer, layer_type):
import re
if layer_type.lower() == "deconvolution" or layer_type.lower(
) == "convolutiondepthwise":
param_name = '_'.join(('convolution', 'param'))
elif layer_type.lower() == "normalize":
param_name = '_'.join(('norm', 'param'))
elif len(layer_type) - len(re.sub("[A-Z]", "", layer_type)) >= 2:
s = ''
tmp_name = ''
for i, ch in enumerate(layer_type):
if i == 0:
s += ch.lower()
continue
elif ch.isupper() and layer_type[i - 1].islower():
tmp_name += (s + '_')
s = ''
s += ch.lower()
tmp_name += s
param_name = '_'.join((tmp_name, 'param'))
else:
param_name = '_'.join((layer_type.lower(), 'param'))
return getattr(layer, param_name, None)
def compute_output_shape(node):
""" compute the output shape of custom layer
"""
layer_type = node.layer_type
assert layer_type in custom_layers, "layer[%s] not exist in custom layers" % (
layer_type)
shape_func = custom_layers[layer_type]['shape']
layer = node.layer
params = get_params(layer, layer_type)
arg_names, kwargs = set_args(shape_func, params)
input_shape = node.input_shape
return shape_func(input_shape, **kwargs)
def make_custom_layer(node):
""" get the code which implement the custom layer function
"""
layer_type = node.layer_type
assert layer_type in custom_layers, "layer[%s] not exist in custom layers" % (
layer_type)
layer_func = custom_layers[layer_type]['layer']
import inspect
return inspect.getsource(layer_func), layer_func
def deal_weights(node, data=None):
""" deal the weights of the custom layer
"""
layer_type = node.layer_type
weights_func = custom_layers[layer_type]['weights']
name = node.layer_name
return weights_func(name, data)
x2paddle/x2paddle/op_mapper/caffe_custom_layer/axpy.py 已删除 100644 → 0
浏览文件 @ c66440ea
from .register import register
from x2paddle.core.util import *
def axpy_shape(input_shapes):
assert len(input_shapes) == 3, "not valid input shape for axpy layer"
assert len(input_shapes[0]) == len(input_shapes[1]), 'should have same dims'
output_shape = input_shapes[1]
assert (input_shapes[2] == output_shape),\
"shape not consistent for axpy[%s <--> %s]" \
% (str(output_shape), str(input_shapes[2]))
return [output_shape]
def axpy_layer(inputs, input_shape=None, name=None):
alpha = inputs[0]
x = inputs[1]
y = inputs[2]
out = fluid.layers.elementwise_mul(x, alpha, axis=0)
out = fluid.layers.elementwise_add(out, y, name=name)
return out
def axpy_weights(name, data=None):
weights_name = []
return weights_name
register(kind='Axpy', shape=axpy_shape, layer=axpy_layer, weights=axpy_weights)
x2paddle/x2paddle/op_mapper/caffe_custom_layer/convolutiondepthwise.py 已删除 100644 → 0
浏览文件 @ c66440ea
from .register import register
from x2paddle.core.util import *
import numbers
def convolutiondepthwise_shape(input_shape,
num_output=None,
pad=None,
kernel_size=None,
stride=None,
dilation=None,
pad_h=None,
pad_w=None,
kernel_h=None,
kernel_w=None,
stride_h=None,
stride_w=None):
[k_h, k_w] = [1, 1]
if isinstance(kernel_size, numbers.Number):
[k_h, k_w] = [kernel_size] * 2
elif len(kernel_size) > 0:
k_h = kernel_h if kernel_h > 0 else kernel_size[0]
k_w = kernel_w if kernel_w > 0 else kernel_size[len(kernel_size) - 1]
elif kernel_h > 0 or kernel_w > 0:
k_h = kernel_h
k_w = kernel_w
[s_h, s_w] = [1, 1]
if isinstance(stride, numbers.Number):
[s_h, s_w] = [stride] * 2
elif len(stride) > 0:
s_h = stride_h if stride_h > 0 else stride[0]
s_w = stride_w if stride_w > 0 else stride[len(stride) - 1]
elif stride_h > 0 or stride_w > 0:
s_h = stride_h
s_w = stride_w
[p_h, p_w] = [0, 0]
if isinstance(pad, numbers.Number):
[p_h, p_w] = [pad] * 2
elif len(pad) > 0:
p_h = pad_h if pad_h > 0 else pad[0]
p_w = pad_w if pad_w > 0 else pad[len(pad) - 1]
elif pad_h > 0 or pad_w > 0:
p_h = pad_h
p_w = pad_w
dila_len = len(dilation)
dila_h = 1
dila_w = 1
if dila_len == 2:
dila_h = dilation[0]
dila_w = dilation[1]
elif dila_len == 1:
dila_h = dila_w = dilation[0]
else:
assert dila_len == 0, "invalid length[%s] of dilation in convolution" % (
dila_len)
i_w = input_shape[0][2]
i_h = input_shape[0][3]
o_h = (i_h + 2 * p_h - (dila_h * (k_h - 1) + 1)) / float(s_h) + 1
o_w = (i_w + 2 * p_w - (dila_w * (k_w - 1) + 1)) / float(s_w) + 1
import math
o_h = int(math.floor(o_h))
o_w = int(math.floor(o_w))
c = num_output if num_output is not None else input_shape[0][1]
return [[input_shape[0][0], c, o_h, o_w]]
def convolutiondepthwise_layer(inputs,
num_output=None,
pad=None,
kernel_size=None,
stride=None,
dilation=None,
pad_h=None,
pad_w=None,
kernel_h=None,
kernel_w=None,
stride_h=None,
stride_w=None,
input_shape=None,
name=None):
import numbers
[k_h, k_w] = [1, 1]
if isinstance(kernel_size, numbers.Number):
[k_h, k_w] = [kernel_size] * 2
elif len(kernel_size) > 0:
k_h = kernel_h if kernel_h > 0 else kernel_size[0]
k_w = kernel_w if kernel_w > 0 else kernel_size[len(kernel_size) - 1]
elif kernel_h > 0 or kernel_w > 0:
k_h = kernel_h
k_w = kernel_w
[s_h, s_w] = [1, 1]
if isinstance(stride, numbers.Number):
[s_h, s_w] = [stride] * 2
elif len(stride) > 0:
s_h = stride_h if stride_h > 0 else stride[0]
s_w = stride_w if stride_w > 0 else stride[len(stride) - 1]
elif stride_h > 0 or stride_w > 0:
s_h = stride_h
s_w = stride_w
[p_h, p_w] = [0, 0]
if isinstance(pad, numbers.Number):
[p_h, p_w] = [pad] * 2
elif len(pad) > 0:
p_h = pad_h if pad_h > 0 else pad[0]
p_w = pad_w if pad_w > 0 else pad[len(pad) - 1]
elif pad_h > 0 or pad_w > 0:
p_h = pad_h
p_w = pad_w
input = inputs[0]
dila_len = len(dilation)
dila_h = 1
dila_w = 1
if dila_len == 2:
dila_h = dilation[0]
dila_w = dilation[1]
elif dila_len == 1:
dila_h = dila_w = dilation[0]
else:
assert dila_len == 0, "invalid length[%s] of dilation in convolution" % (
dila_len)
c_in = input_shape[0][1]
c_out = num_output if num_output is not None else input_shape[0][1]
group = int(c_in / (c_in / c_out)) if c_in > c_out else int(c_in /
(c_out / c_in))
out = fluid.layers.conv2d(
input,
dilation=[dila_h, dila_w],
filter_size=[k_h, k_w],
stride=[s_h, s_w],
padding=[p_h, p_w],
groups=group,
num_filters=c_out,
param_attr=name + '_weights',
bias_attr=name + '_bias',
name=name)
return out
def convolutiondepthwise_weights(name, data=None):
weights_name = []
weights_name.append(name + '_weights')
weights_name.append(name + '_bias')
return weights_name
register(
kind='ConvolutionDepthwise',
shape=convolutiondepthwise_shape,
layer=convolutiondepthwise_layer,
weights=convolutiondepthwise_weights)
x2paddle/x2paddle/op_mapper/caffe_custom_layer/detectionoutput.py 已删除 100644 → 0
浏览文件 @ c66440ea
from .register import register
from x2paddle.core.util import *
def detectionoutput_shape(input_shape):
return [[-1, 6]]
def detectionoutput_layer(inputs,
nms_param=None,
background_label_id=0,
share_location=True,
keep_top_k=100,
confidence_threshold=0.1,
input_shape=None,
name=None):
nms_param_str = nms_param
nms_param = {}
part = nms_param_str.split(',')
for s in part:
if s == '':
break
else:
name, obj = s.split(': ')
if name == 'top_k':
nms_param[name] = int(obj)
else:
nms_param[name] = float(obj)
if nms_param is None:
nms_param = {"nms_threshold": 0.3, "top_k": 10, "eta": 1.0}
mbox_conf_flatten = inputs[1]
mbox_priorbox = inputs[2]
mbox_priorbox_list = fluid.layers.split(mbox_priorbox, 2, dim=1)
pb = mbox_priorbox_list[0]
pbv = mbox_priorbox_list[1]
pb = fluid.layers.reshape(x=pb, shape=[-1, 4])
pbv = fluid.layers.reshape(x=pbv, shape=[-1, 4])
mbox_loc = inputs[0]
mbox_loc = fluid.layers.reshape(x=mbox_loc, shape=[-1, pb.shape[0], 4])
mbox_conf_flatten = fluid.layers.reshape(
x=mbox_conf_flatten, shape=[0, pb.shape[0], -1])
default = {"nms_threshold": 0.3, "top_k": 10, "eta": 1.0}
fields = ['eta', 'top_k', 'nms_threshold']
for f in default.keys():
if f not in nms_param:
nms_param[f] = default[f]
out = fluid.layers.detection_output(
scores=mbox_conf_flatten,
loc=mbox_loc,
prior_box=pb,
prior_box_var=pbv,
background_label=background_label_id,
nms_threshold=nms_param["nms_threshold"],
nms_top_k=nms_param["top_k"],
keep_top_k=keep_top_k,
score_threshold=confidence_threshold,
nms_eta=nms_param["eta"])
return out
def detectionoutput_weights(name, data=None):
weights_name = []
return weights_name
register(
kind='DetectionOutput',
shape=detectionoutput_shape,
layer=detectionoutput_layer,
weights=detectionoutput_weights)
x2paddle/x2paddle/op_mapper/caffe_custom_layer/normalize.py 已删除 100644 → 0
浏览文件 @ c66440ea
from .register import register
from x2paddle.core.util import *
def normalize_shape(input_shape):
return input_shape
def normalize_layer(inputs,
across_spatial=None,
channel_shared=None,
input_shape=None,
name=None):
assert across_spatial == False, "Only support across_spatial == False for Normalize"
input = inputs[0]
l2_norm = fluid.layers.l2_normalize(input, axis=1, name=name + '_l2')
scale_param = fluid.layers.create_parameter(
shape=[1] if channel_shared else [1, 1, 1, input_shape[0][1]],
dtype=input.dtype,
attr=fluid.ParamAttr(name=name + '_scale'))
scale_param = fluid.layers.reshape(x=scale_param, \
shape=[1] if channel_shared else [input_shape[0][1]])
out = fluid.layers.elementwise_mul(
x=l2_norm, y=scale_param, axis=-1 if channel_shared else 1)
return out
def normalize_weights(name, data=None):
weights_name = [name + '_scale']
return weights_name
register(
kind='Normalize',
shape=normalize_shape,
layer=normalize_layer,
weights=normalize_weights)
x2paddle/x2paddle/op_mapper/caffe_custom_layer/permute.py 已删除 100644 → 0
浏览文件 @ c66440ea
from .register import register
from x2paddle.core.util import *
def permute_shape(input_shape, order=None):
inshape = input_shape[0]
output_shape = []
for ii in order:
assert ii < len(inshape), "invalid order for permute[%s]" % (name)
output_shape.append(inshape[ii])
return [output_shape]
def permute_layer(inputs, order=None, input_shape=None, name=None):
input = inputs[0]
order = list(order)
out = fluid.layers.transpose(input, perm=order, name=name)
return out
def permute_weights(name, data=None):
weights_name = []
return weights_name
register(
kind='Permute',
shape=permute_shape,
layer=permute_layer,
weights=permute_weights)
x2paddle/x2paddle/op_mapper/caffe_custom_layer/priorbox.py 已删除 100644 → 0
浏览文件 @ c66440ea
from .register import register
from x2paddle.core.util import *
def priorbox_shape(input_shape, max_size=None, aspect_ratio=None):
fc_shape = input_shape[0]
N = 1
if not max_size == None:
N += 1
if not aspect_ratio == None:
N += 2 * len(aspect_ratio)
N_bbx = fc_shape[2] * fc_shape[3] * N
output_shape = [1, 2, 4 * N_bbx]
return [output_shape]
def priorbox_layer(inputs,
step=0.0,
offset=0.5,
min_size=None,
max_size=[],
aspect_ratio=[1.0],
flip=False,
clip=False,
variance=[0.1, 0.1, 0.2, 0.2],
input_shape=None,
name=None):
input = inputs[0]
image = inputs[1]
steps = tuple(step) if type(step) is list or type(step) is tuple else (step,
step)
box, variance_ = fluid.layers.prior_box(
input,
image,
min_sizes=min_size,
max_sizes=max_size,
aspect_ratios=aspect_ratio,
variance=variance,
flip=flip,
clip=clip,
steps=steps,
offset=offset,
name=name,
min_max_aspect_ratios_order=True)
box = fluid.layers.reshape(box, [1, 1, -1])
variance_ = fluid.layers.reshape(variance_, [1, 1, -1])
out = fluid.layers.concat([box, variance_], axis=1)
return out
def priorbox_weights(name, data=None):
weights_name = []
return weights_name
register(
kind='PriorBox',
shape=priorbox_shape,
layer=priorbox_layer,
weights=priorbox_weights)
x2paddle/x2paddle/op_mapper/caffe_custom_layer/register.py 已删除 100644 → 0
浏览文件 @ c66440ea
""" this module provides 'register' for registering customized layers
"""
g_custom_layers = {}
def register(kind, shape, layer, weights):
""" register a custom layer or a list of custom layers
Args:
@kind (str or list): type name of the layer
@shape (function): a function to generate the shape of layer's output
@layer (function): a function to generate the paddle code of layer
@weights (function): a function to deal with weights data
Returns:
None
"""
assert type(shape).__name__ == 'function', 'shape should be a function'
assert type(layer).__name__ == 'function', 'layer should be a function'
if type(kind) is str:
kind = [kind]
else:
assert type(
kind) is list, 'invalid param "kind" for register, not a list or str'
for k in kind:
assert type(
k) is str, 'invalid param "kind" for register, not a list of str'
assert k not in g_custom_layers, 'this type[%s] has already been registered' % (
k)
g_custom_layers[k] = {
'shape': shape,
'layer': layer,
'weights': weights
}
def get_registered_layers():
return g_custom_layers
x2paddle/x2paddle/op_mapper/caffe_custom_layer/relu6.py 已删除 100644 → 0
浏览文件 @ c66440ea
from .register import register
from x2paddle.core.util import *
def relu6_shape(input_shape):
return input_shape
def relu6_layer(inputs, input_shape=None, name=None):
input = inputs[0]
out = fluid.layers.relu6(x=input)
return out
def relu6_weights(name, data=None):
weights_name = []
return weights_name
register(
kind='ReLU6', shape=relu6_shape, layer=relu6_layer, weights=relu6_weights)
x2paddle/x2paddle/op_mapper/caffe_custom_layer/roipooling.py 已删除 100644 → 0
浏览文件 @ c66440ea
from .register import register
from x2paddle.core.util import *
def roipooling_shape(input_shape, pooled_w=None, pooled_h=None):
base_fea_shape = input_shapes[0]
rois_shape = input_shapes[1]
output_shape = base_fea_shape
output_shape[0] = rois_shape[0]
output_shape[2] = pooled_h
output_shape[3] = pooled_w
return [output_shape]
def roipooling_layer(inputs,
pooled_w=None,
pooled_h=None,
spatial_scale=None,
input_shape=None,
name=None):
input = inputs[0]
roi = inputs[1]
roi = fluid.layers.slice(roi, axes=[1], starts=[1], ends=[5])
out = fluid.layers.roi_pool(
input,
roi,
pooled_height=pooled_h,
pooled_width=pooled_w,
spatial_scale=spatial_scale)
return out
def roipooling_weights(name, data=None):
weights_name = []
return weights_name
register(
kind='ROIPooling',
shape=roipooling_shape,
layer=roipooling_layer,
weights=roipooling_weights)
x2paddle/x2paddle/op_mapper/caffe_custom_layer/select.py 已删除 100644 → 0
浏览文件 @ c66440ea
from .register import register
from x2paddle.core.util import *
def select_shape(input_shape, axis=None, slice_point=None):
inshape = input_shape[0]
slice_point = slice_point
start = slice_point[0]
if len(slice_point) == 2:
end = slice_point[1]
else:
end = input_shape[axis]
assert end > start, "invalid slice_point with [start:%d, end:%d]" % (start,
end)
output_shape = input_shape
output_shape[axis] = end - start
return [output_shape]
def select_layer(inputs,
axis=None,
slice_point=None,
input_shape=None,
name=None):
input = inputs[0]
maxint32 = 2147483647
slice_point = [0] + slice_point
slice_point.append(maxint32)
i = 0
out = []
for i in range(len(slice_point)):
out.append(
fluid.layers.slice(
input,
axes=[axis],
starts=[slice_point[i]],
ends=[slice_point[i + 1]],
name=name + '_' + str(i)))
if i == len(slice_point) - 2:
break
return out
def select_weights(name, data=None):
weights_name = []
return weights_name
register(
kind='Select',
shape=select_shape,
layer=select_layer,
weights=select_weights)
x2paddle/x2paddle/op_mapper/caffe_custom_layer/shufflechannel.py 已删除 100644 → 0
浏览文件 @ c66440ea
from .register import register
from x2paddle.core.util import *
def shufflechannel_shape(input_shape):
return input_shape
def shufflechannel_layer(inputs, group=None, input_shape=None, name=None):
input = inputs[0]
out = fluid.layers.shuffle_channel(x=input, group=group)
return out
def shufflechannel_weights(name, data=None):
weights_name = []
return weights_name
register(
kind='ShuffleChannel',
shape=shufflechannel_shape,
layer=shufflechannel_layer,
weights=shufflechannel_weights)
x2paddle/x2paddle/op_mapper/caffe_custom_layer/upsample.py 已删除 100644 → 0
浏览文件 @ c66440ea
# -*- coding: utf-8 -*-
################################################################################
#
# Copyright (c) 2020 Baidu.com, Inc. All Rights Reserved
#
################################################################################
"""
Author: Drift
Email: wutuobang@baidu.com
Date: 2020/04/22 18:45
"""
from .register import register
from x2paddle.core.util import *
def upsample_shape(input_shapes, scale):
"""
:param input_shapes:
:param scale:
:return:
"""
assert len(input_shapes) == 1, "not valid input shape for upsample layer"
assert type(scale) is int
input_shape = input_shapes[0]
new_h = scale * input_shape[2]
new_w = scale * input_shape[3]
output_shape = [input_shape[0], input_shape[1], new_h, new_w]
return [output_shape]
def upsample_layer(inputs, scale, input_shape=None, name=None):
"""
:param inputs:
:param scale:
:param input_shape:
:param name:
:return:
"""
x = inputs[0]
out = fluid.layers.resize_nearest(
x, align_corners=False, scale=scale, name=name)
return out
def upsample_weights(name, data=None):
"""
:param name:
:param data:
:return:
"""
weights_name = []
return weights_name
register(
kind='Upsample',
shape=upsample_shape,
layer=upsample_layer,
weights=upsample_weights)
x2paddle/x2paddle/op_mapper/caffe_shape.py 已删除 100644 → 0
浏览文件 @ c66440ea
此差异已折叠。
x2paddle/x2paddle/op_mapper/onnx2paddle/opset9/__init__.py 已删除 100644 → 0
浏览文件 @ c66440ea
from .opset import OpSet9
from .custom_layer import custom_layers
x2paddle/x2paddle/op_mapper/tf_op_mapper.py 已删除 100644 → 0
浏览文件 @ c66440ea
此差异已折叠。
x2paddle/x2paddle/optimizer/onnx_optimizer.py 已删除 100644 → 0
浏览文件 @ c66440ea
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# TODO useless node remove
class ONNXOptimizer(object):
def __init__(self, op_mapper):
self.op_mapper = op_mapper
self.graph = op_mapper.graph
def delete_redundance_code(self):
for node_name in self.graph.topo_sort:
if node_name in self.op_mapper.omit_nodes:
node = self.graph.get_node(node_name)
omit_freq = self.op_mapper.omit_nodes.count(node_name)
if len(node.outputs) <= omit_freq:
node.fluid_code.clear()
x2paddle/x2paddle/optimizer/tf_optimizer.py 已删除 100644 → 0
浏览文件 @ c66440ea
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或