diff --git a/.travis.yml b/.travis.yml index 69cd7c720dffc742458dba0bb432794b160026e4..7f181a905b9ae093bc183890ac3ef0b167fd0cdf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: - '3.6' script: - - if [[ $TRAVIS_PYTHON_VERSION != 2.7 ]]; then /bin/bash ./scripts/check_code_style.sh; fi + - if [[ $TRAVIS_PYTHON_VERSION != 2.7 ]]; then /bin/bash ./tools/check_code_style.sh; fi notifications: email: diff --git a/README.md b/README.md index b81f2ce511d7ffe254eba24aeb07e6dd93f12e73..b9efad3bf26280364794e0f78325d10058a8e3d4 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ X2Paddle在多个主流的CV模型上，测试过TensorFlow/Caffe/ONNX模型的 ## 环境依赖 -python >= 3.5 +python == 2.7 | python >= 3.5 paddlepaddle >= 1.5.0 **按需安装以下依赖** diff --git a/scripts/check_code_style.sh b/tools/check_code_style.sh similarity index 100% rename from scripts/check_code_style.sh rename to tools/check_code_style.sh diff --git a/tools/check_for_lite.py b/tools/check_for_lite.py index 94581b20ccdd43fd4f5b39c7210b3a74b2e273d1..78fb55f5ee309106daf12d805a2efa95dcdaaad2 100644 --- a/tools/check_for_lite.py +++ b/tools/check_for_lite.py @@ -1,4 +1,4 @@ -import urllib +from six.moves import urllib import sys from paddle.fluid.framework import Program diff --git a/x2paddle/__init__.py b/x2paddle/__init__.py index 98a433b31058a4c8166312666d91533a8e53e50a..3d187266f146fb216a04537da37dc5df0811acc4 100644 --- a/x2paddle/__init__.py +++ b/x2paddle/__init__.py @@ -1 +1 @@ -__version__ = "0.4.5" +__version__ = "0.5.0" diff --git a/x2paddle/core/fluid_code.py b/x2paddle/core/fluid_code.py index 42b51f117143cd7a3f7b3a579d2ba6c4e2f33a74..5619e087260397128de3e12563ed1cf9c776f945 100644 --- a/x2paddle/core/fluid_code.py +++ b/x2paddle/core/fluid_code.py @@ -13,8 +13,9 @@ # limitations under the License. from x2paddle.core.graph import GraphNode -import collections from x2paddle.core.util import * +import collections +import six class Layer(object): @@ -28,7 +29,7 @@ class Layer(object): def get_code(self): layer_code = "" if self.output is not None: - if isinstance(self.output, str): + if isinstance(self.output, six.string_types): layer_code = self.output + " = " else: layer_code = self.output.layer_name + " = " @@ -47,7 +48,7 @@ class Layer(object): "[{}]".format(input.index) + ", ") else: in_list += (input.layer_name + ", ") - elif isinstance(input, str): + elif isinstance(input, six.string_types): in_list += (input + ", ") else: raise Exception( @@ -72,7 +73,7 @@ class Layer(object): "[{}]".format(self.inputs.index) + ", ") else: layer_code += (self.inputs.layer_name + ", ") - elif isinstance(self.inputs, str): + elif isinstance(self.inputs, six.string_types): layer_code += (self.inputs + ", ") else: raise Exception("Unknown type of inputs.") @@ -119,6 +120,6 @@ class FluidCode(object): for layer in self.layers: if isinstance(layer, Layer): codes.append(layer.get_code()) - elif isinstance(layer, str): + elif isinstance(layer, six.string_types): codes.append(layer) return codes diff --git a/x2paddle/core/graph.py b/x2paddle/core/graph.py index 3165dcd1d9ea26ff39408c3628de8ba982c4657b..06e585e711d10727415b2a2b1ab872883657aabd 100644 --- a/x2paddle/core/graph.py +++ b/x2paddle/core/graph.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function +from __future__ import division import collections import copy as cp @@ -98,8 +100,3 @@ class Graph(object): raise Exception("node[{}] not in graph".format(dst)) self.node_map[dst].inputs.append(src) self.node_map[src].outputs.append(dst) - - def print(self): - for i, tmp in enumerate(self.topo_sort): - print(tmp, self.node_map[tmp].layer_type, self.node_map[tmp].inputs, - self.node_map[tmp].outputs) diff --git a/x2paddle/decoder/caffe_decoder.py b/x2paddle/decoder/caffe_decoder.py index 9573d10409064245fe33338d5ac8394604487ae1..ea9d3bc76c06f976b082838b1d794e95ac3488d0 100644 --- a/x2paddle/decoder/caffe_decoder.py +++ b/x2paddle/decoder/caffe_decoder.py @@ -236,11 +236,7 @@ class CaffeDecoder(object): data.MergeFromString(open(self.model_path, 'rb').read()) pair = lambda layer: (layer.name, self.normalize_pb_data(layer)) layers = data.layers or data.layer - import time - start = time.time() self.params = [pair(layer) for layer in layers if layer.blobs] - end = time.time() - print('cost:', str(end - start)) def normalize_pb_data(self, layer): transformed = [] diff --git a/x2paddle/op_mapper/caffe_op_mapper.py b/x2paddle/op_mapper/caffe_op_mapper.py index 1d451639e7063b72b888ecf913c9d2be9cc8bb3e..3e3f249f2e75ace83876d5154aa1f9a034f1b668 100644 --- a/x2paddle/op_mapper/caffe_op_mapper.py +++ b/x2paddle/op_mapper/caffe_op_mapper.py @@ -954,6 +954,13 @@ class CaffeOpMapper(OpMapper): inputs_node = [] for i in range(len(node.inputs)): input = self.graph.get_bottom_node(node, idx=i, copy=True) + if i == 1 and op == 'DetectionOutput': + input = self.graph.get_bottom_node(node, idx=i, copy=True) + print(input.layer_type) + while input is not None and input.layer_type != 'Softmax': + input = self.graph.get_bottom_node(input, idx=0, copy=True) + assert input is not None, 'This kind of DetectionOutput is not supported!' + input = self.graph.get_bottom_node(input, idx=0, copy=True) inputs_node.append(input) node.fluid_code.add_layer(func.__code__.co_name, inputs=inputs_node, diff --git a/x2paddle/op_mapper/onnx_op_mapper.py b/x2paddle/op_mapper/onnx_op_mapper.py index a2d8941a3276eb164bb496f6f0e7a6f4448265d3..4dcc922c15d21612d409dd2e568ca83335c7e194 100644 --- a/x2paddle/op_mapper/onnx_op_mapper.py +++ b/x2paddle/op_mapper/onnx_op_mapper.py @@ -94,7 +94,7 @@ class ONNXOpMapper(OpMapper): print(op) return False - def directly_map(self, node, *args, name='', **kwargs): + def directly_map(self, node, name='', *args, **kwargs): inputs = node.layer.input outputs = node.layer.output op_type = node.layer_type diff --git a/x2paddle/op_mapper/tf_op_mapper.py b/x2paddle/op_mapper/tf_op_mapper.py index 12784189d6c38daa406b0b8dfb9d4a32d825dfbe..73b0da50d797e2eb9fca0c724499e0ed4fdbf367 100644 --- a/x2paddle/op_mapper/tf_op_mapper.py +++ b/x2paddle/op_mapper/tf_op_mapper.py @@ -24,6 +24,8 @@ import sys def get_same_padding(in_size, kernel_size, stride): new_size = int(math.ceil(in_size * 1.0 / stride)) pad_size = (new_size - 1) * stride + kernel_size - in_size + if pad_size < 0: + pad_size = 0 pad0 = int(pad_size / 2) pad1 = pad_size - pad0 return [pad0, pad1] @@ -369,12 +371,13 @@ class TFOpMapper(OpMapper): pad_w = get_same_padding(in_shape[3], k_size[3], strides[3]) pad_h = pad_h[0] + pad_h[1] pad_w = pad_w[0] + pad_w[1] - attr = {"paddings": [0, pad_h, 0, pad_w], "pad_value": -10000.0} - node.fluid_code.add_layer("pad2d", - inputs=input, - output=node, - param_attr=attr) - input = node + if pad_h != 0 or pad_w != 0: + attr = {"paddings": [0, pad_h, 0, pad_w], "pad_value": -10000.0} + node.fluid_code.add_layer("pad2d", + inputs=input, + output=node, + param_attr=attr) + input = node attr = { "pool_size": k_size[2:4], "pool_type": string("max"), @@ -551,6 +554,7 @@ class TFOpMapper(OpMapper): def Reshape(self, node): input = self.graph.get_node(node.layer.input[0], copy=True) param = self.graph.get_node(node.layer.input[1], copy=True) + is_variable = False if param.layer_type == "Const": attr = {"shape": param.value.tolist()} self.add_omit_nodes(param.layer_name, node.layer_name) @@ -582,6 +586,24 @@ class TFOpMapper(OpMapper): new_param += (node.layer_name + "[{}]".format(i) + ", ") new_param = new_param.strip(", ") + "]" attr = {"shape": new_param} + is_variable = True + + # to change [192, -1]->[-1, 192], allways put -1 in the first dimension + # optimization for Paddle-Lite + in_shape = input.out_shapes[0] + if is_variable and in_shape.count(-1) < 1: + total_size = 1 + for i in range(len(in_shape)): + total_size *= in_shape[i] + for i in range(len(attr["shape"])): + if attr["shape"][i] == 0: + attr["shape"][i] = in_shape[i] + if attr["shape"][i] != -1: + total_size /= attr["shape"][i] + if attr["shape"].count(-1) > 0: + index = attr["shape"].index(-1) + attr["shape"][index] = int(total_size) + attr["shape"][0] = -1 if len(input.out_shapes[0]) == 4 and node.tf_data_format == "NHWC": if len(attr["shape"]) < 3: @@ -763,6 +785,9 @@ class TFOpMapper(OpMapper): start = self.graph.get_node(node.layer.input[0], copy=True) limit = self.graph.get_node(node.layer.input[1], copy=True) delta = self.graph.get_node(node.layer.input[2], copy=True) + self.add_omit_nodes(start.layer_name, node.layer_name) + self.add_omit_nodes(limit.layer_name, node.layer_name) + self.add_omit_nodes(delta.layer_name, node.layer_name) if start.layer_type == "Const": start = start.value else: @@ -775,9 +800,6 @@ class TFOpMapper(OpMapper): delta = delta.value else: delta = self.decoder.infer_tensor(delta) - self.add_omit_nodes(start.layer_name, node.layer_name) - self.add_omit_nodes(limit.layer_name, node.layer_name) - self.add_omit_nodes(delta.layer_name, node.layer_name) inputs = {"start": start, "end": limit, "step": delta} attr = {"dtype": string(node.dtype)} diff --git a/x2paddle/op_mapper/tf_op_mapper_nhwc.py b/x2paddle/op_mapper/tf_op_mapper_nhwc.py index 4412d897d6f3e06429bb8418c452851f8863d764..9399c6d7a13f26fee17af9a76a52f6d5de5dba8b 100644 --- a/x2paddle/op_mapper/tf_op_mapper_nhwc.py +++ b/x2paddle/op_mapper/tf_op_mapper_nhwc.py @@ -24,6 +24,8 @@ import sys def get_same_padding(in_size, kernel_size, stride): new_size = int(math.ceil(in_size * 1.0 / stride)) pad_size = (new_size - 1) * stride + kernel_size - in_size + if pad_size < 0: + pad_size = 0 pad0 = int(pad_size / 2) pad1 = pad_size - pad0 return [pad0, pad1] @@ -500,6 +502,7 @@ class TFOpMapperNHWC(OpMapper): def Reshape(self, node): input = self.graph.get_node(node.layer.input[0], copy=True) param = self.graph.get_node(node.layer.input[1], copy=True) + is_variable = False if param.layer_type == "Const": attr = {"shape": param.value.tolist()} self.add_omit_nodes(param.layer_name, node.layer_name) @@ -527,6 +530,24 @@ class TFOpMapperNHWC(OpMapper): new_param += (node.layer_name + "[{}]".format(i) + ", ") new_param = new_param.strip(", ") + "]" attr = {"shape": new_param} + is_variable = True + # to change [192, -1]->[-1, 192], allways put -1 in the first dimension + # optimization for Paddle-Lite + in_shape = input.out_shapes[0] + if not is_variable and in_shape.count(-1) < 1: + total_size = 1 + for i in range(len(in_shape)): + total_size *= in_shape[i] + for i in range(len(attr["shape"])): + if attr["shape"][i] == 0: + attr["shape"][i] = in_shape[i] + if attr["shape"][i] != -1: + total_size /= attr["shape"][i] + if attr["shape"].count(-1) > 0: + index = attr["shape"].index(-1) + attr["shape"][index] = int(total_size) + attr["shape"][0] = -1 + node.fluid_code.add_layer("reshape", inputs=input, output=node,