Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
未来在这儿
X2Paddle
提交
c0327eca
X
X2Paddle
项目概览
未来在这儿
/
X2Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / X2Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
X
X2Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
c0327eca
编写于
8月 08, 2019
作者:
J
Jason
提交者:
GitHub
8月 08, 2019
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #82 from SunAhong1993/develop
only for caffe.proto
上级
d4c8f5f6
7118b2ff
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
87 addition
and
129 deletion
+87
-129
x2paddle/convert.py
x2paddle/convert.py
+0
-18
x2paddle/decoder/caffe_decoder.py
x2paddle/decoder/caffe_decoder.py
+84
-104
x2paddle/op_mapper/caffe_op_mapper.py
x2paddle/op_mapper/caffe_op_mapper.py
+3
-7
未找到文件。
x2paddle/convert.py
浏览文件 @
c0327eca
...
...
@@ -91,24 +91,6 @@ def tf2paddle(model_path, save_dir):
def
caffe2paddle
(
proto
,
weight
,
save_dir
,
caffe_proto
):
if
caffe_proto
is
not
None
:
import
os
if
caffe_proto
is
not
None
and
not
os
.
path
.
isfile
(
caffe_proto
):
print
(
"The .py file compiled by caffe.proto is not exist."
)
return
else
:
try
:
import
caffe
version
=
caffe
.
__version__
if
version
!=
'1.0.0'
:
print
(
"caffe == 1.0.0 is required"
)
return
except
:
print
(
"Caffe is not installed."
)
print
(
"You have 2 options: 1. install caffe 2. compile the caffe.proto"
)
return
from
x2paddle.decoder.caffe_decoder
import
CaffeDecoder
from
x2paddle.op_mapper.caffe_op_mapper
import
CaffeOpMapper
...
...
x2paddle/decoder/caffe_decoder.py
浏览文件 @
c0327eca
...
...
@@ -23,43 +23,28 @@ from x2paddle.op_mapper import caffe_shape
class
CaffeResolver
(
object
):
def
__init__
(
self
,
caffe_proto
):
self
.
proto_path
=
caffe_proto
if
self
.
proto_path
is
None
:
self
.
use_default
=
True
else
:
self
.
use_default
=
False
self
.
caffe_proto
=
caffe_proto
self
.
import_caffe
()
def
import_caffepb
(
self
):
(
filepath
,
tempfilename
)
=
os
.
path
.
split
(
os
.
path
.
abspath
(
self
.
proto_path
))
(
filename
,
extension
)
=
os
.
path
.
splitext
(
tempfilename
)
sys
.
path
.
append
(
filepath
)
out
=
__import__
(
filename
)
if
self
.
caffe_proto
is
None
:
from
x2paddle.decoder
import
caffe_pb2
out
=
caffe_pb2
else
:
if
not
os
.
path
.
isfile
(
self
.
caffe_proto
):
raise
Exception
(
"The .py file compiled by caffe.proto is not exist."
)
(
filepath
,
tempfilename
)
=
os
.
path
.
split
(
os
.
path
.
abspath
(
self
.
caffe_proto
))
(
filename
,
extension
)
=
os
.
path
.
splitext
(
tempfilename
)
sys
.
path
.
append
(
filepath
)
out
=
__import__
(
filename
)
return
out
def
import_caffe
(
self
):
self
.
caffe
=
None
self
.
caffepb
=
None
if
self
.
use_default
:
try
:
# Try to import PyCaffe first
import
caffe
self
.
caffe
=
caffe
except
ImportError
:
# Fall back to the protobuf implementation
self
.
caffepb
=
self
.
import_caffepb
()
else
:
self
.
caffepb
=
self
.
import_caffepb
()
if
self
.
caffe
:
# Use the protobuf code from the imported distribution.
# This way, Caffe variants with custom layers will work.
self
.
caffepb
=
self
.
caffe
.
proto
.
caffe_pb2
self
.
caffepb
=
self
.
import_caffepb
()
self
.
NetParameter
=
self
.
caffepb
.
NetParameter
def
has_pycaffe
(
self
):
return
self
.
caffe
is
not
None
class
CaffeGraphNode
(
GraphNode
):
def
__init__
(
self
,
layer
,
layer_name
=
None
):
...
...
@@ -125,62 +110,6 @@ class CaffeGraph(Graph):
layers
=
self
.
model
.
layers
or
self
.
model
.
layer
layers
=
self
.
filter_layers
(
layers
)
inputs_num
=
len
(
self
.
model
.
input
)
if
inputs_num
!=
0
:
input_dims_num
=
len
(
self
.
model
.
input_dim
)
if
input_dims_num
!=
0
:
if
input_dims_num
>
0
and
input_dims_num
!=
inputs_num
*
4
:
raise
Error
(
'invalid input_dim[%d] param in prototxt'
%
(
input_dims_num
))
for
i
in
range
(
inputs_num
):
dims
=
self
.
model
.
input_dim
[
i
*
4
:(
i
+
1
)
*
4
]
data
=
self
.
model
.
layer
.
add
()
try
:
from
caffe
import
layers
as
L
data
.
CopyFrom
(
L
.
Input
(
input_param
=
dict
(
shape
=
dict
(
dim
=
[
dims
[
0
],
dims
[
1
],
dims
[
2
],
dims
[
3
]
]))).
to_proto
().
layer
[
0
])
except
:
print
(
"The .py file compiled by .proto file does not work for the old style prototxt. "
)
print
(
"There are 2 solutions for you as below:"
)
print
(
"1. install caffe and don
\'
t set
\'
--caffe_proto
\'
."
)
print
(
"2. modify your .prototxt from the old style to the new style."
)
sys
.
exit
(
-
1
)
data
.
name
=
self
.
model
.
input
[
i
]
data
.
top
[
0
]
=
self
.
model
.
input
[
i
]
else
:
for
i
in
range
(
inputs_num
):
dims
=
self
.
model
.
input_shape
[
i
].
dim
[
0
:
4
]
data
=
self
.
model
.
layer
.
add
()
try
:
from
caffe
import
layers
as
L
data
.
CopyFrom
(
L
.
Input
(
input_param
=
dict
(
shape
=
dict
(
dim
=
[
dims
[
0
],
dims
[
1
],
dims
[
2
],
dims
[
3
]
]))).
to_proto
().
layer
[
0
])
except
:
print
(
"The .py file compiled by .proto file does not work for the old style prototxt. "
)
print
(
"There are 2 solutions for you as below:"
)
print
(
"1. install caffe and don
\'
t set
\'
--caffe_proto
\'
."
)
print
(
"2. modify your .prototxt from the old style to the new style."
)
sys
.
exit
(
-
1
)
data
.
name
=
self
.
model
.
input
[
i
]
data
.
top
[
0
]
=
self
.
model
.
input
[
i
]
layers
=
[
data
]
+
layers
top_layer
=
{}
for
layer
in
layers
:
self
.
node_map
[
layer
.
name
]
=
CaffeGraphNode
(
layer
)
...
...
@@ -202,7 +131,7 @@ class CaffeGraph(Graph):
node
=
self
.
node_map
[
layer_name
]
node
.
set_params
(
data
)
else
:
raise
Exception
(
'Ignoring parameters for non-existent layer: %s'
%
\
print
(
'Ignoring parameters for non-existent layer: %s'
%
\
layer_name
)
super
(
CaffeGraph
,
self
).
build
()
...
...
@@ -221,35 +150,20 @@ class CaffeGraph(Graph):
class
CaffeDecoder
(
object
):
def
__init__
(
self
,
proto_path
,
model_path
,
caffe_proto
=
None
):
def
__init__
(
self
,
proto_path
,
model_path
,
caffe_proto
):
self
.
proto_path
=
proto_path
self
.
model_path
=
model_path
self
.
resolver
=
CaffeResolver
(
caffe_proto
=
caffe_proto
)
self
.
net
=
self
.
resolver
.
NetParameter
()
with
open
(
proto_path
,
'rb'
)
as
proto_file
:
proto_str
=
proto_file
.
read
(
)
proto_str
=
self
.
old2new
(
proto_file
)
text_format
.
Merge
(
proto_str
,
self
.
net
)
self
.
load
()
self
.
load
_using_pb
()
self
.
caffe_graph
=
CaffeGraph
(
self
.
net
,
self
.
params
)
self
.
caffe_graph
.
build
()
def
load
(
self
):
if
self
.
resolver
.
has_pycaffe
():
self
.
load_using_caffe
()
else
:
self
.
load_using_pb
()
def
load_using_caffe
(
self
):
caffe
=
self
.
resolver
.
caffe
caffe
.
set_mode_cpu
()
print
(
self
.
proto_path
)
print
(
self
.
model_path
)
net
=
caffe
.
Net
(
self
.
proto_path
,
self
.
model_path
,
caffe
.
TEST
)
data
=
lambda
blob
:
blob
.
data
self
.
params
=
[(
k
,
list
(
map
(
data
,
v
)))
for
k
,
v
in
net
.
params
.
items
()]
def
load_using_pb
(
self
):
data
=
self
.
resolver
.
NetParameter
()
data
.
MergeFromString
(
open
(
self
.
model_path
,
'rb'
).
read
())
...
...
@@ -271,3 +185,69 @@ class CaffeDecoder(object):
data
=
np
.
array
(
blob
.
data
,
dtype
=
np
.
float32
).
reshape
(
c_o
,
c_i
,
h
,
w
)
transformed
.
append
(
data
)
return
transformed
def
old2new
(
self
,
proto_file
):
part1_str
=
''
part2_str
=
''
part3_str
=
''
is_input
=
False
dims
=
[]
line
=
proto_file
.
readline
()
print
(
'Check if it is a new style of caffe...'
)
while
line
:
l_str
=
bytes
.
decode
(
line
)
if
l_str
.
replace
(
' '
,
''
).
startswith
(
'input:'
):
part2_str
+=
'layer {
\n
'
part2_str
+=
(
' name: '
+
l_str
.
strip
().
replace
(
' '
,
''
).
split
(
'input:'
)[
-
1
]
+
'
\n
'
)
part2_str
+=
' type:
\"
Input
\"\n
'
part2_str
+=
(
' top: '
+
l_str
.
strip
().
replace
(
' '
,
''
).
split
(
'input:'
)[
-
1
]
+
'
\n
'
)
is_input
=
True
line
=
proto_file
.
readline
()
continue
elif
l_str
.
replace
(
' '
,
''
).
startswith
(
'input_dim:'
):
dims
.
append
(
int
(
l_str
.
strip
().
replace
(
' '
,
''
).
split
(
'input_dim:'
)[
-
1
]))
if
len
(
dims
)
==
4
:
part2_str
+=
' input_param { shape: { dim: '
+
str
(
dims
[
0
])
+
\
' dim: '
+
str
(
dims
[
1
])
+
\
' dim: '
+
str
(
dims
[
2
])
+
\
' dim: '
+
str
(
dims
[
3
])
+
' } }
\n
'
dims
=
[]
part2_str
+=
'}
\n
'
line
=
proto_file
.
readline
()
if
bytes
.
decode
(
line
).
replace
(
' '
,
''
).
startswith
(
'}'
):
line
=
proto_file
.
readline
()
continue
elif
l_str
.
replace
(
' '
,
''
).
startswith
(
'input_shape'
):
part2_str
+=
l_str
.
replace
(
'input_shape'
,
'input_param { shape: '
)
l_str
=
bytes
.
decode
(
proto_file
.
readline
())
while
l_str
:
if
'}'
in
l_str
:
part2_str
+=
l_str
+
'
\n
}
\n
}'
break
else
:
part2_str
+=
l_str
l_str
=
bytes
.
decode
(
proto_file
.
readline
())
line
=
proto_file
.
readline
()
continue
if
not
is_input
:
part1_str
+=
bytes
.
decode
(
line
)
else
:
part3_str
+=
bytes
.
decode
(
line
)
line
=
proto_file
.
readline
()
out
=
part1_str
+
part2_str
+
part3_str
layer_str
=
'layer{'
part
=
out
.
split
(
layer_str
)
if
len
(
part
)
==
1
:
layer_str
=
'layer {'
part
=
out
.
split
(
layer_str
)
for
i
in
range
(
len
(
part
)):
if
part
[
i
].
strip
().
replace
(
' '
,
''
)
==
''
or
part
[
i
].
count
(
':'
)
>
1
:
continue
out
=
out
.
replace
(
layer_str
+
part
[
i
],
part
[
i
].
replace
(
' '
,
''
))
return
str
.
encode
(
out
)
x2paddle/op_mapper/caffe_op_mapper.py
浏览文件 @
c0327eca
...
...
@@ -27,10 +27,6 @@ class CaffeOpMapper(OpMapper):
self
.
weights
=
dict
()
resolver
=
decoder
.
resolver
self
.
used_custom_layers
=
{}
if
resolver
.
has_pycaffe
():
self
.
did_use_pb
=
False
else
:
self
.
did_use_pb
=
True
print
(
"Total nodes: {}"
.
format
(
len
(
self
.
graph
.
topo_sort
)))
for
node_name
in
self
.
graph
.
topo_sort
:
...
...
@@ -79,8 +75,6 @@ class CaffeOpMapper(OpMapper):
def
adjust_parameters
(
self
,
node
):
data
=
node
.
data
if
not
self
.
did_use_pb
:
return
data
# When using the protobuf-backend, each parameter initially has four dimensions.
# In certain cases (like FC layers), we want to eliminate the singleton dimensions.
# This implementation takes care of the common cases. However, it does leave the
...
...
@@ -93,6 +87,8 @@ class CaffeOpMapper(OpMapper):
squeeze_indices
.
append
(
0
)
# Squeeze FC.
for
idx
in
squeeze_indices
:
print
(
'Transform the weights of {}...'
.
format
(
node
.
layer_name
+
str
(
idx
)))
if
idx
>=
len
(
data
):
continue
...
...
@@ -892,7 +888,7 @@ class CaffeOpMapper(OpMapper):
output
=
node
,
param_attr
=
attr
)
def
Flatten
(
self
,
no
ed
):
def
Flatten
(
self
,
no
de
):
assert
len
(
node
.
inputs
)
==
1
,
'The count of DetectionOutput node
\'
s input is not 1.'
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录