未验证 提交 8c2b2e97 编写于 作者: J Jason 提交者: GitHub

Merge pull request #86 from SunAhong1993/develop

add directly map op and custom layer readme
## 如何转换Caffe自定义Layer
本文档介绍如何将Caffe自定义Layer转换为PaddlePaddle模型中的对应实现, 用户可根据自己需要,添加代码实现自定义层,从而支持模型的完整转换。
***步骤一 下载代码***
此处涉及修改源码,应先卸载x2paddle,并且下载源码,主要有以下两步完成:
```
pip uninstall x2paddle
pip install git+https://github.com/PaddlePaddle/X2Paddle.git@develop
```
***步骤二 编译caffe.proto***
该步骤依赖protobuf编译器,其安装过程有以下两种方式:
> 选择一:pip install protobuf
> 选择二:使用[官方源码](https://github.com/protocolbuffers/protobuf)进行编译
使用脚本./tools/compile.sh将caffe.proto(包含所需的自定义Layer信息)编译成我们所需的目标语言(Python)
使用方式:
```
bash ./toos/compile.sh /home/root/caffe/src/caffe/proto
# /home/root/caffe/src/caffe/proto为caffe.proto的存放路径,生成的caffe_pb2.py也将保存在该路径下
```
***步骤三 添加自定义Layer的实现代码***
- 进入./x2paddle/op_mapper/caffe_custom_layer,创建.py文件,例如mylayer.py
- 仿照./x2paddle/op_mapper/caffe_custom_layer中的其他文件,在mylayer.py中主要需要实现3个函数,下面以roipooling.py为例分析代码:
1. `def roipooling_shape(input_shape, pooled_w=None, pooled_h=None)`
参数:
1. input_shape(list):其中每个元素代表该层每个输入数据的shape,为必须传入的参数
2. pooled_w(int):代表ROI Pooling的kernel的宽,其命名与.prototxt中roi_pooling_param中的key一致
3. pooled_h(int):代表ROI Pooling的kernel的高,其命名与.prototxt中roi_pooling_param中的key一致
功能:计算出进行ROI Pooling后的shape
返回:一个list,其中每个元素代表每个输出数据的shape,由于ROI Pooling的输出数据只有一个,所以其list长度为1
2. `def roipooling_layer(inputs, input_shape=None, name=None, pooled_w=None, pooled_h=None, spatial_scale=None)`
参数:
1. inputs(list):其中每个元素代表该层每个输入数据,为必须传入的参数
2. input_shape(list):其中每个元素代表该层每个输入数据的shape,为必须传入的参数
3. name(str):ROI Pooling层的名字,为必须传入的参数
4. pooled_w(int):代表ROI Pooling的kernel的宽,其命名与.prototxt中roi_pooling_param中的key一致
5. pooled_h(int):代表ROI Pooling的kernel的高,其命名与.prototxt中roi_pooling_param中的key一致
6. spatial_scale(float):用于将ROI坐标从输入比例转换为池化时使用的比例,其命名与.prototxt中roi_pooling_param中的key一致
功能:运用PaddlePaddle完成组网来实现`roipooling_layer`的功能
返回:一个Variable,为组网后的结果
3. `def roipooling_weights(name, data=None)`
参数:
1. name(str):ROI Pooling层的名字,为必须传入的参数
2. data(list):由Caffe模型.caffemodel获得的关于roipooling的参数,roipooling的参数为None
功能:为每个参数(例如kernel、bias等)命名;同时,若Caffe中该层参数与PaddlePaddle中参数的格式不一致,则变换操作也在该函数中实现。
返回:一个list,包含每个参数的名字。
- 在roipooling.py中注册`roipooling`,主要运用下述代码实现:
```
register(kind='ROIPooling', shape=roipooling_shape, layer=roipooling_layer, weights=roipooling_weights)
# kind为在model.prototxt中roipooling的type
```
- 在./x2paddle/op_mapper/caffe_custom_layer/\_\_init\_\_.py中引入该层的使用
```
from . import roipooling
```
***步骤四 运行转换代码***
```
# 在X2Paddle目录下安装x2paddle
python setup.py install
# 运行转换代码
x2paddle --framework=caffe
--prototxt=deploy.proto
--weight=deploy.caffemodel
--save_dir=pd_model
--caffe_proto=/home/root/caffe/src/caffe/proto/caffe_pb2.py
```
#!/bin/bash
#function:
# script used to generate caffe_pb2.py from caffe.proto using protoc
#
PROTOC=`which protoc`
if [[ -z $PROTOC ]];then
echo "not found protoc, you should first install it following this[https://github.com/google/protobuf/releases]"
exit 1
fi
WORK_ROOT=$1
PY_NAME="$WORK_ROOT/caffe_pb2.py"
$PROTOC --proto_path=$WORK_ROOT --python_out=$WORK_ROOT $WORK_ROOT/caffe.proto
ret=$?
if [ -e "$PY_NAME" ];then
echo "succeed to generate [$PY_NAME]"
exit 0
else
echo "failed to generate [$PY_NAME]"
fi
exit $ret
......@@ -22,6 +22,13 @@ from x2paddle.op_mapper.caffe_custom_layer import *
class CaffeOpMapper(OpMapper):
directly_map_ops = {
'ReLU': 'relu',
'AbsVal': 'abs',
'Sigmoid': 'sigmoid',
'TanH': 'tanh',
}
def __init__(self, decoder):
super(CaffeOpMapper, self).__init__()
self.graph = decoder.caffe_graph
......@@ -40,8 +47,12 @@ class CaffeOpMapper(OpMapper):
elif op in custom_layers:
self.set_node_shape(node, is_fluid_op=False)
self.deal_custom_layer(node)
elif op in self.directly_map_ops:
self.set_node_shape(node)
self.directly_map(node)
else:
raise Exception("Model are not supported yet.")
raise Exception(
"The op {} in model is not supported yet.".format(op))
def op_checker(self):
unsupported_ops = set()
......@@ -180,6 +191,22 @@ class CaffeOpMapper(OpMapper):
output=node,
param_attr=attr)
def MemoryData(self, node):
# TODO(syf): Paddlepaddle can't fully support
shape = node.output_shape[0][1:]
dtype = 'float32'
attr = {
'dtype': string(dtype),
'shape': shape,
'name': string(node.layer_name)
}
node.fluid_code.add_layer("data",
inputs=None,
output=node.layer_name + '0',
param_attr=attr)
node.fluid_code.add_note('{} = [{}]'.format(node.layer_name,
node.layer_name + '0'))
def Convolution(self, node):
data = node.data
assert data is not None, 'The parameter of {} (type is {}) is not set. You need to use python package of caffe to set the default value.'.format(
......@@ -290,16 +317,6 @@ class CaffeOpMapper(OpMapper):
output=node,
param_attr=attr)
def ReLU(self, node):
assert len(
node.inputs) == 1, 'The count of ReLU node\'s input is not 1.'
input = self.graph.get_bottom_node(node, idx=0, copy=True)
attr = {'name': string(node.layer_name)}
node.fluid_code.add_layer("relu",
inputs=input,
output=node,
param_attr=attr)
def LRN(self, node):
assert len(node.inputs) == 1, 'The count of LRN node\'s input is not 1.'
params = node.layer.lrn_param
......@@ -445,26 +462,6 @@ class CaffeOpMapper(OpMapper):
output=node,
param_attr=attr)
def Sigmoid(self, node):
assert len(
node.inputs) == 1, 'The count of PReLU node\'s input is not 1.'
input = self.graph.get_bottom_node(node, idx=0, copy=True)
attr = {'name': string(node.layer_name)}
node.fluid_code.add_layer("sigmoid",
inputs=input,
output=node,
param_attr=attr)
def AbsVal(self, node):
assert len(
node.inputs) == 1, 'The count of PReLU node\'s input is not 1.'
input = self.graph.get_bottom_node(node, idx=0, copy=True)
attr = {'name': string(node.layer_name)}
node.fluid_code.add_layer("absval",
inputs=input,
output=node,
param_attr=attr)
def Accuracy(self, node):
assert len(
node.inputs) == 2, 'The count of Accuracy node\'s input is not 2.'
......@@ -492,16 +489,6 @@ class CaffeOpMapper(OpMapper):
output=node,
param_attr=attr)
def TanH(self, node):
assert len(
node.inputs) == 1, 'The count of TanH node\'s input is not 1.'
input = self.graph.get_bottom_node(node, idx=0, copy=True)
attr = {'name': string(node.layer_name)}
node.fluid_code.add_layer("tanh",
inputs=input,
output=node,
param_attr=attr)
def Eltwise(self, node):
assert len(
node.inputs) == 2, 'The count of TanH node\'s input is not 2.'
......@@ -892,3 +879,13 @@ class CaffeOpMapper(OpMapper):
is_custom_layer=True)
if op not in self.used_custom_layers:
self.used_custom_layers[op] = custom_code
def directly_map(self, node):
assert node.layer_type in self.directly_map_ops
op_info = self.directly_map_ops[node.layer_type]
input = self.graph.get_bottom_node(node, idx=0, copy=True)
attr = {'name': string(node.layer_name)}
node.fluid_code.add_layer(op_info,
inputs=input,
output=node,
param_attr=attr)
......@@ -74,11 +74,12 @@ def shape_convolution(layer, input_shape):
def shape_deconvolution(layer, input_shape):
h_i = input_shape[2]
w_i = input_shape[3]
h_i = input_shape[0][2]
w_i = input_shape[0][3]
params = layer.convolution_param
dila_h, dila_w, pad_h, pad_w, kernel_h, kernel_w, stride_h, stride_w = get_params_w_h(
dila_h, dila_w, pad_h, pad_w, kernel_h, kernel_w, stride_h, stride_w = get_kernel_parameters(
params)
h_o = (h_i - 1) * stride_h - 2 * pad_h + dila_h * (kernel_h - 1) + 1
......@@ -124,6 +125,16 @@ def shape_input(layer, input_shape):
return [list(layer.input_param.shape[0].dim)]
def shape_memorydata(layer, input_shape):
params = layer.memory_data_param
shape = []
shape.append(int(params.batch_size))
shape.append(int(params.channels))
shape.append(int(params.height))
shape.append(int(params.width))
return [shape]
def shape_concat(layer, input_shape):
params = layer.concat_param
axis = params.axis
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册