Merge pull request #5 from PaddlePaddle/develop

Develop

README.md

@@ -6,7 +6,7 @@ X2Paddle is a toolkit for converting trained model to PaddlePaddle from other de
 
 python >= 3.5  
 paddlepaddle >= 1.5.0  
-tensorflow == 1.x  
+tensorflow == 1.14.0  
 
 ## 安装
 ```
@@ -20,17 +20,23 @@ x2paddle --framework=tensorflow --model=tf_model.pb --save_dir=pd_model
 ```
 ### Caffe
 ```
-x2paddle --framework=caffe --proto=deploy.proto --weight=deploy.caffemodel --save_dir=pd_model
+x2paddle --framework=caffe --prototxt=deploy.proto --weight=deploy.caffemodel --save_dir=pd_model
 ```
 ### 参数选项
 | 参数 | |
 |----------|--------------|
 |--framework | 源模型类型 (tensorflow、caffe) |
-|--proto | 当framework为caffe时，该参数指定caffe模型的proto文件路径 |
+|--prototxt | 当framework为caffe时，该参数指定caffe模型的proto文件路径 |
 |--weight | 当framework为caffe时，该参数指定caffe模型的参数文件路径 |
 |--save_dir | 指定转换后的模型保存目录路径 |
 |--model | 当framework为tensorflow时，该参数指定tensorflow的pb模型文件路径 |
+|--caffe_proto | [可选]由caffe.proto编译成caffe_pb2.py文件的存放路径，当没有安装caffe或者使用自定义Layer时使用 |
+
+## 使用转换后的模型
+转换后的模型包括`model_with_code`和`inference_model`两个目录。  
+`model_with_code`中保存了模型参数，和转换后的python模型代码  
+`inference_model`中保存了序列化的模型结构和参数，可直接使用paddle的接口进行加载，见[load_inference_model](https://www.paddlepaddle.org.cn/documentation/docs/zh/1.5/api_guides/low_level/inference.html#api-guide-inference)
 
 ## Related Docs
 1. [X2Paddle使用过程中常见问题](Q&A.md)  
-2. [如何导出TensorFlow的pb模型](export_tf_model.md)
\ No newline at end of file
+2. [如何导出TensorFlow的pb模型](export_tf_model.md)

tool/tf_export_model.py

-#   Copyright (c) 2019  PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from tensorflow.python.framework import graph_util
-import tensorflow as tf
-
-
-def freeze_model(sess, output_tensor_names, freeze_model_path):
-    out_graph = graph_util.convert_variables_to_constants(
-        sess, sess.graph.as_graph_def(), output_tensor_names)
-    with tf.gfile.GFile(freeze_model_path, 'wb') as f:
-        f.write(out_graph.SerializeToString())
-
-    print("freeze model saved in {}".format(freeze_model_path))
-
-
-import tensorflow.contrib.slim as slim
-from tensorflow.contrib.slim.nets import vgg
-import numpy
-
-with tf.Session() as sess:
-    inputs = tf.placeholder(dtype=tf.float32,
-                            shape=[None, None, None, 3],
-                            name="inputs")
-    logits, endpoint = vgg.vgg_16(inputs, num_classes=1000, is_training=False)
-    load_model = slim.assign_from_checkpoint_fn(
-        "vgg_16.ckpt", slim.get_model_variables("vgg_16"))
-    load_model(sess)
-
-    numpy.random.seed(13)
-    data = numpy.random.rand(5, 224, 224, 3)
-    input_tensor = sess.graph.get_tensor_by_name("inputs:0")
-    output_tensor = sess.graph.get_tensor_by_name("vgg_16/fc8/squeezed:0")
-    result = sess.run([output_tensor], {input_tensor: data})
-    numpy.save("tensorflow.npy", numpy.array(result))
-
-    freeze_model(sess, ["vgg_16/fc8/squeezed"], "vgg16_None.pb")

x2paddle/convert.py

@@ -63,7 +63,7 @@ def tf2paddle(model_path, save_dir):
             )
             return
     except:
-        print("Tensorflow not installed, use \"pip install tensorflow\"")
+        print("Tensorflow is not installed, use \"pip install tensorflow\".")
         return
 
     from x2paddle.decoder.tf_decoder import TFDecoder
@@ -83,6 +83,21 @@ def tf2paddle(model_path, save_dir):
 
 
 def caffe2paddle(proto, weight, save_dir, caffe_proto):
+    try:
+        import caffe
+        version = caffe.__version__
+        if version == '1.0.0':
+            print("caffe == 1.0.0 is required")
+            return
+    except:
+        print("Caffe is not installed.")
+        import os
+        if not os.path.isfile(caffe_proto + 'caffe_pb2.py'):
+            print("The file that resolve caffe is not exist.")
+            print(
+                "You have 2 options: 1. install caffe 2. compile the caffe.proto"
+            )
+            return
     from x2paddle.decoder.caffe_decoder import CaffeDecoder
     from x2paddle.op_mapper.caffe_op_mapper import CaffeOpMapper
 

x2paddle/core/graph.py

@@ -49,11 +49,13 @@ class Graph(object):
 
     def _make_input_nodes(self):
         for name, node in self.node_map.items():
+            name = name.replace('/', '_').replace('-', '_')
             if len(node.inputs) == 0:
                 self.input_nodes.append(name)
 
     def _make_output_nodes(self):
         for name, node in self.node_map.items():
+            name = name.replace('/', '_').replace('-', '_')
             if len(node.outputs) == 0:
                 self.output_nodes.append(name)
 

x2paddle/decoder/caffe_decoder.py

@@ -97,7 +97,6 @@ class CaffeGraph(Graph):
         phase_map = {0: 'train', 1: 'test'}
         filtered_layer_names = set()
         filtered_layers = []
-        print('The filter layer:')
         for layer in layers:
             phase = 'test'
             if len(layer.include):
@@ -116,7 +115,7 @@ class CaffeGraph(Graph):
                 assert layer.name not in filtered_layer_names
                 filtered_layer_names.add(layer.name)
             else:
-                print(layer.name)
+                print('The filter layer:' + layer.name)
         return filtered_layers
 
     def build(self):
@@ -184,8 +183,8 @@ class CaffeGraph(Graph):
                 node = self.node_map[layer_name]
                 node.set_params(data)
             else:
-                notice('Ignoring parameters for non-existent layer: %s' % \
-                        layer_name)
+                raise Exception('Ignoring parameters for non-existent layer: %s' % \
+                       layer_name)
 
         super(CaffeGraph, self).build()
 

x2paddle/op_mapper/caffe_custom_layer/__init__.py

@@ -56,7 +56,7 @@ def get_params(layer, layer_type):
             elif ch.isupper() and layer_type[i - 1].islower():
                 tmp_name += (s + '_')
                 s = ''
-            s += ch
+            s += ch.lower()
         tmp_name += s
         param_name = '_'.join((tmp_name, 'param'))
     else:

x2paddle/op_mapper/caffe_custom_layer/convolutiondepthwise.py

@@ -18,19 +18,19 @@ def convolutiondepthwise_shape(input_shape,
     [k_h, k_w] = [1, 1]
     if isinstance(kernel_size, numbers.Number):
         [k_h, k_w] = [kernel_size] * 2
-    elif isinstance(kernel_size, list) and len(kernel_size) > 0:
+    elif len(kernel_size) > 0:
         k_h = kernel_h if kernel_h else kernel_size[0]
         k_w = kernel_w if kernel_w else kernel_size[len(kernel_size) - 1]
     [s_h, s_w] = [1, 1]
     if isinstance(stride, numbers.Number):
         [s_h, s_w] = [stride] * 2
-    elif isinstance(stride, list) and len(stride) > 0:
+    elif len(stride) > 0:
         s_h = stride_h if stride_h else stride[0]
         s_w = stride_w if stride_w else stride[len(stride) - 1]
     [p_h, p_w] = [0, 0]
     if isinstance(pad, numbers.Number):
         [p_h, p_w] = [pad] * 2
-    elif isinstance(pad, list) and len(pad) > 0:
+    elif len(pad) > 0:
         p_h = pad_h if pad_h else pad[0]
         p_w = pad_w if pad_w else pad[len(pad) - 1]
     dila_len = len(dilation)
@@ -69,22 +69,23 @@ def convolutiondepthwise_layer(inputs,
                                stride_w=None,
                                input_shape=None,
                                name=None):
+    import numbers
     [k_h, k_w] = [1, 1]
     if isinstance(kernel_size, numbers.Number):
         [k_h, k_w] = [kernel_size] * 2
-    elif isinstance(kernel_size, list) and len(kernel_size) > 0:
+    elif len(kernel_size) > 0:
         k_h = kernel_h if kernel_h else kernel_size[0]
         k_w = kernel_w if kernel_w else kernel_size[len(kernel_size) - 1]
     [s_h, s_w] = [1, 1]
     if isinstance(stride, numbers.Number):
         [s_h, s_w] = [stride] * 2
-    elif isinstance(stride, list) and len(stride) > 0:
+    elif len(stride) > 0:
         s_h = stride_h if stride_h else stride[0]
         s_w = stride_w if stride_w else stride[len(stride) - 1]
     [p_h, p_w] = [0, 0]
     if isinstance(pad, numbers.Number):
         [p_h, p_w] = [pad] * 2
-    elif isinstance(pad, list) and len(pad) > 0:
+    elif len(pad) > 0:
         p_h = pad_h if pad_h else pad[0]
         p_w = pad_w if pad_w else pad[len(pad) - 1]
     input = inputs[0]

x2paddle/op_mapper/caffe_op_mapper.py

@@ -123,22 +123,21 @@ class CaffeOpMapper(OpMapper):
         [k_h, k_w] = [1, 1]
         if isinstance(params.kernel_size, numbers.Number):
             [k_h, k_w] = [params.kernel_size] * 2
-        elif isinstance(params.kernel_size,
-                        list) and len(params.kernel_size) > 0:
+        elif len(params.kernel_size) > 0:
             k_h = params.kernel_h if params.kernel_h else params.kernel_size[0]
             k_w = params.kernel_w if params.kernel_w else params.kernel_size[
                 len(params.kernel_size) - 1]
         [s_h, s_w] = [1, 1]
         if isinstance(params.stride, numbers.Number):
             [s_h, s_w] = [params.stride] * 2
-        elif isinstance(params.stride, list) and len(params.stride) > 0:
+        elif len(params.stride) > 0:
             s_h = params.stride_h if params.stride_h else params.stride[0]
             s_w = params.stride_w if params.stride_w else params.stride[
                 len(params.stride) - 1]
         [p_h, p_w] = [0, 0]
         if isinstance(params.pad, numbers.Number):
             [p_h, p_w] = [params.pad] * 2
-        elif isinstance(params.pad, list) and len(params.pad) > 0:
+        elif len(params.pad) > 0:
             p_h = params.pad_h if params.pad_h else params.pad[0]
             p_w = params.pad_w if params.pad_w else params.pad[len(params.pad) -
                                                                1]