Merge pull request #4 from PaddlePaddle/develop

pull

caffe_custom_layer.md

@@ -10,5 +10,3 @@
 | Normalize | [code](https://github.com/weiliu89/caffe/blob/ssd/src/caffe/layers/normalize_layer.cpp) |
 | ROIPooling | [code](https://github.com/rbgirshick/caffe-fast-rcnn/blob/0dcd397b29507b8314e252e850518c5695efbb83/src/caffe/layers/roi_pooling_layer.cpp) |
 | Axpy | [code](https://github.com/hujie-frank/SENet/blob/master/src/caffe/layers/axpy_layer.cpp) |
-
-

scripts/check_code_style.sh

@@ -7,7 +7,6 @@ function abort(){
 
 trap 'abort' 0
 set -e
-
 cd $TRAVIS_BUILD_DIR
 export PATH=/usr/bin:$PATH
 pre-commit install

x2paddle/convert.py

@@ -179,6 +179,9 @@ def main():
             x2paddle.__version__))
         return
 
+    assert args.framework is not None, "--framework is not defined(support tensorflow/caffe/onnx)"
+    assert args.save_dir is not None, "--save_dir is not defined"
+
     try:
         import paddle
         v0, v1, v2 = paddle.__version__.split('.')
@@ -187,8 +190,6 @@ def main():
             return
     except:
         print("paddlepaddle not installed, use \"pip install paddlepaddle\"")
-    assert args.framework is not None, "--framework is not defined(support tensorflow/caffe/onnx)"
-    assert args.save_dir is not None, "--save_dir is not defined"
 
     if args.framework == "tensorflow":
         assert args.model is not None, "--model should be defined while translating tensorflow model"

x2paddle/core/fluid_code.py

@@ -13,8 +13,9 @@
 # limitations under the License.
 
 from x2paddle.core.graph import GraphNode
-import collections
 from x2paddle.core.util import *
+import collections
+import six
 
 
 class Layer(object):
@@ -28,7 +29,7 @@ class Layer(object):
     def get_code(self):
         layer_code = ""
         if self.output is not None:
-            if isinstance(self.output, str):
+            if isinstance(self.output, six.string_types):
                 layer_code = self.output + " = "
             else:
                 layer_code = self.output.layer_name + " = "
@@ -47,7 +48,7 @@ class Layer(object):
                                     "[{}]".format(input.index) + ", ")
                     else:
                         in_list += (input.layer_name + ", ")
-                elif isinstance(input, str):
+                elif isinstance(input, six.string_types):
                     in_list += (input + ", ")
                 else:
                     raise Exception(
@@ -72,7 +73,7 @@ class Layer(object):
                                "[{}]".format(self.inputs.index) + ", ")
             else:
                 layer_code += (self.inputs.layer_name + ", ")
-        elif isinstance(self.inputs, str):
+        elif isinstance(self.inputs, six.string_types):
             layer_code += (self.inputs + ", ")
         else:
             raise Exception("Unknown type of inputs.")
@@ -119,6 +120,6 @@ class FluidCode(object):
         for layer in self.layers:
             if isinstance(layer, Layer):
                 codes.append(layer.get_code())
-            elif isinstance(layer, str):
+            elif isinstance(layer, six.string_types):
                 codes.append(layer)
         return codes

x2paddle/core/graph.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+from __future__ import division
 import collections
 import copy as cp
 

x2paddle/core/op_mapper.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import paddle.fluid as fluid
 from paddle.fluid.proto import framework_pb2
 from x2paddle.core.util import *
 import inspect
@@ -46,6 +47,28 @@ def export_paddle_param(param, param_name, dir):
     fp.close()
 
 
+# This func will copy to generate code file
+def run_net(param_dir="./"):
+    import os
+    inputs, outputs = x2paddle_net()
+    for i, out in enumerate(outputs):
+        if isinstance(out, list):
+            for out_part in out:
+                outputs.append(out_part)
+            del outputs[i]
+    exe = fluid.Executor(fluid.CPUPlace())
+    exe.run(fluid.default_startup_program())
+
+    def if_exist(var):
+        b = os.path.exists(os.path.join(param_dir, var.name))
+        return b
+
+    fluid.io.load_vars(exe,
+                       param_dir,
+                       fluid.default_main_program(),
+                       predicate=if_exist)
+
+
 class OpMapper(object):
     def __init__(self):
         self.paddle_codes = ""

x2paddle/core/util.py

@@ -11,8 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-import paddle.fluid as fluid
 import numpy
 import math
 import os
@@ -20,25 +18,3 @@ import os
 
 def string(param):
     return "\'{}\'".format(param)
-
-
-# This func will copy to generate code file
-def run_net(param_dir="./"):
-    import os
-    inputs, outputs = x2paddle_net()
-    for i, out in enumerate(outputs):
-        if isinstance(out, list):
-            for out_part in out:
-                outputs.append(out_part)
-            del outputs[i]
-    exe = fluid.Executor(fluid.CPUPlace())
-    exe.run(fluid.default_startup_program())
-
-    def if_exist(var):
-        b = os.path.exists(os.path.join(param_dir, var.name))
-        return b
-
-    fluid.io.load_vars(exe,
-                       param_dir,
-                       fluid.default_main_program(),
-                       predicate=if_exist)

x2paddle/decoder/caffe_decoder.py

@@ -236,11 +236,7 @@ class CaffeDecoder(object):
         data.MergeFromString(open(self.model_path, 'rb').read())
         pair = lambda layer: (layer.name, self.normalize_pb_data(layer))
         layers = data.layers or data.layer
-        import time
-        start = time.time()
         self.params = [pair(layer) for layer in layers if layer.blobs]
-        end = time.time()
-        print('cost:', str(end - start))
 
     def normalize_pb_data(self, layer):
         transformed = []

x2paddle/op_mapper/caffe_custom_layer/normalize.py

@@ -18,6 +18,8 @@ def normalize_layer(inputs,
         shape=[1] if channel_shared else [input_shape[0][1]],
         dtype=input.dtype,
         attr=name + '_scale')
+    scale_param = fluid.layers.reshape(x=scale_param, \
+                  shape=[1] if channel_shared else [input_shape[0][1]])
     out = fluid.layers.elementwise_mul(x=l2_norm,
                                        y=scale_param,
                                        axis=-1 if channel_shared else 1)

x2paddle/op_mapper/caffe_op_mapper.py

@@ -135,7 +135,8 @@ class CaffeOpMapper(OpMapper):
         if isinstance(params.kernel_size, numbers.Number):
             [k_h, k_w] = [params.kernel_size] * 2
         elif len(params.kernel_size) > 0:
-            k_h = params.kernel_h if params.kernel_h > 0 else params.kernel_size[0]
+            k_h = params.kernel_h if params.kernel_h > 0 else params.kernel_size[
+                0]
             k_w = params.kernel_w if params.kernel_w > 0 else params.kernel_size[
                 len(params.kernel_size) - 1]
         elif params.kernel_h > 0 or params.kernel_w > 0:
@@ -156,8 +157,8 @@ class CaffeOpMapper(OpMapper):
             [p_h, p_w] = [params.pad] * 2
         elif len(params.pad) > 0:
             p_h = params.pad_h if params.pad_h > 0 else params.pad[0]
-            p_w = params.pad_w if params.pad_w > 0 else params.pad[len(params.pad) -
-                                                                  1]
+            p_w = params.pad_w if params.pad_w > 0 else params.pad[
+                len(params.pad) - 1]
         elif params.pad_h > 0 or params.pad_w > 0:
             p_h = params.pad_h
             p_w = params.pad_w
@@ -225,12 +226,17 @@ class CaffeOpMapper(OpMapper):
             node.layer_type, params)
         if data is None:
             data = []
-            print('The parameter of {} (type is {}) is not set. So we set the parameters as 0'.format(
-            node.layer_name, node.layer_type))
+            print(
+                'The parameter of {} (type is {}) is not set. So we set the parameters as 0'
+                .format(node.layer_name, node.layer_type))
             input_c = node.input_shape[0][1]
             output_c = channel
-            data.append(np.zeros([output_c, input_c, kernel[0], kernel[1]]).astype('float32'))
-            data.append(np.zeros([output_c,])).astype('float32')
+            data.append(
+                np.zeros([output_c, input_c, kernel[0],
+                          kernel[1]]).astype('float32'))
+            data.append(np.zeros([
+                output_c,
+            ])).astype('float32')
         else:
             data = self.adjust_parameters(node)
         self.weights[node.layer_name + '_weights'] = data[0]
@@ -272,12 +278,17 @@ class CaffeOpMapper(OpMapper):
             node.layer_type, params)
         if data is None:
             data = []
-            print('The parameter of {} (type is {}) is not set. So we set the parameters as 0'.format(
-            node.layer_name, node.layer_type))
+            print(
+                'The parameter of {} (type is {}) is not set. So we set the parameters as 0'
+                .format(node.layer_name, node.layer_type))
             input_c = node.input_shape[0][1]
             output_c = channel
-            data.append(np.zeros([output_c, input_c, kernel[0], kernel[1]]).astype('float32'))
-            data.append(np.zeros([output_c,]).astype('float32'))
+            data.append(
+                np.zeros([output_c, input_c, kernel[0],
+                          kernel[1]]).astype('float32'))
+            data.append(np.zeros([
+                output_c,
+            ]).astype('float32'))
         else:
             data = self.adjust_parameters(node)
         self.weights[node.layer_name + '_weights'] = data[0]
@@ -369,13 +380,17 @@ class CaffeOpMapper(OpMapper):
         data = node.data
         params = node.layer.inner_product_param
         if data is None:
-            print('The parameter of {} (type is {}) is not set. So we set the parameters as 0.'.format(
-            node.layer_name, node.layer_type))
+            print(
+                'The parameter of {} (type is {}) is not set. So we set the parameters as 0.'
+                .format(node.layer_name, node.layer_type))
             input_c = node.input_shape[0][1]
             output_c = params.num_output
             data = []
-            data.append(np.zeros([input_c, output_c]).astype('float32').astype('float32'))
-            data.append(np.zeros([output_c]).astype('float32').astype('float32'))
+            data.append(
+                np.zeros([input_c,
+                          output_c]).astype('float32').astype('float32'))
+            data.append(
+                np.zeros([output_c]).astype('float32').astype('float32'))
         else:
             data = self.adjust_parameters(node)
             # Reshape the parameters to Paddle's ordering
@@ -616,7 +631,8 @@ class CaffeOpMapper(OpMapper):
                                       param_attr=attr)
 
     def BatchNorm(self, node):
-        assert len(node.inputs) == 1, 'The count of BatchNorm node\'s input is not 1.'
+        assert len(
+            node.inputs) == 1, 'The count of BatchNorm node\'s input is not 1.'
         input = self.graph.get_bottom_node(node, idx=0, copy=True)
         params = node.layer.batch_norm_param
         if hasattr(params, 'eps'):
@@ -624,11 +640,16 @@ class CaffeOpMapper(OpMapper):
         else:
             eps = 1e-5
         if node.data is None or len(node.data) != 3:
-            print('The parameter of {} (type is {}) is not set. So we set the parameters as 0'.format(
-            node.layer_name, node.layer_type))
+            print(
+                'The parameter of {} (type is {}) is not set. So we set the parameters as 0'
+                .format(node.layer_name, node.layer_type))
             input_c = node.input_shape[0][1]
-            mean = np.zeros([input_c,]).astype('float32')
-            variance = np.zeros([input_c,]).astype('float32')
+            mean = np.zeros([
+                input_c,
+            ]).astype('float32')
+            variance = np.zeros([
+                input_c,
+            ]).astype('float32')
             scale = 0
         else:
             node.data = [np.squeeze(i) for i in node.data]
@@ -655,11 +676,16 @@ class CaffeOpMapper(OpMapper):
 
     def Scale(self, node):
         if node.data is None:
-            print('The parameter of {} (type is {}) is not set. So we set the parameters as 0'.format(
-            node.layer_name, node.layer_type))
+            print(
+                'The parameter of {} (type is {}) is not set. So we set the parameters as 0'
+                .format(node.layer_name, node.layer_type))
             input_c = node.input_shape[0][1]
-            self.weights[node.layer_name + '_scale'] = np.zeros([input_c,]).astype('float32')
-            self.weights[node.layer_name + '_offset'] = np.zeros([input_c,]).astype('float32')
+            self.weights[node.layer_name + '_scale'] = np.zeros([
+                input_c,
+            ]).astype('float32')
+            self.weights[node.layer_name + '_offset'] = np.zeros([
+                input_c,
+            ]).astype('float32')
         else:
             self.weights[node.layer_name + '_scale'] = np.squeeze(node.data[0])
             self.weights[node.layer_name + '_offset'] = np.squeeze(node.data[1])

x2paddle/op_mapper/caffe_shape.py

@@ -43,7 +43,8 @@ def get_kernel_parameters(params):
         [p_h, p_w] = [params.pad] * 2
     elif len(params.pad) > 0:
         p_h = params.pad_h if params.pad_h > 0 else params.pad[0]
-        p_w = params.pad_w if params.pad_w > 0 else params.pad[len(params.pad) - 1]
+        p_w = params.pad_w if params.pad_w > 0 else params.pad[len(params.pad) -
+                                                               1]
     elif params.pad_h > 0 or params.pad_w > 0:
         p_h = params.pad_h
         p_w = params.pad_w

x2paddle/op_mapper/onnx_op_mapper.py

@@ -94,7 +94,7 @@ class ONNXOpMapper(OpMapper):
                 print(op)
             return False
 
-    def directly_map(self, node, *args, name='', **kwargs):
+    def directly_map(self, node, name='', *args, **kwargs):
         inputs = node.layer.input
         outputs = node.layer.output
         op_type = node.layer_type

x2paddle_model_zoo.md

@@ -39,7 +39,7 @@
 | ResNet50 | [code](https://github.com/soeaver/caffe-model/blob/master/cls/resnet/deploy_resnet50.prototxt) |
 | Unet | [code](https://github.com/jolibrain/deepdetect/blob/master/templates/caffe/unet/deploy.prototxt) |
 | VGGNet | [code](https://gist.github.com/ksimonyan/211839e770f7b538e2d8#file-vgg_ilsvrc_16_layers_deploy-prototxt) |
-
+| FaceDetection | [code](https://github.com/ShiqiYu/libfacedetection/blob/master/models/caffe/yufacedetectnet-open-v1.prototxt) |
 
 
 
@@ -65,4 +65,3 @@
 | mNASNet | [pytorch(personal practice)](https://github.com/rwightman/gen-efficientnet-pytorch) |9|
 | EfficientNet | [pytorch(personal practice)](https://github.com/rwightman/gen-efficientnet-pytorch) |9|
 | SqueezeNet | [onnx official](https://s3.amazonaws.com/download.onnx/models/opset_9/squeezenet.tar.gz) |9|
-