Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
吃玉米的猫
Paddle
提交
812de6e8
P
Paddle
项目概览
吃玉米的猫
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
812de6e8
编写于
8月 14, 2018
作者:
M
minqiyang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Port utils to Python3
上级
6988aea9
变更
11
隐藏空白更改
内联
并排
Showing
11 changed file
with
66 addition
and
53 deletion
+66
-53
python/paddle/utils/dump_config.py
python/paddle/utils/dump_config.py
+2
-2
python/paddle/utils/image_multiproc.py
python/paddle/utils/image_multiproc.py
+9
-7
python/paddle/utils/image_util.py
python/paddle/utils/image_util.py
+4
-4
python/paddle/utils/make_model_diagram.py
python/paddle/utils/make_model_diagram.py
+26
-19
python/paddle/utils/merge_model.py
python/paddle/utils/merge_model.py
+1
-1
python/paddle/utils/plotcurve.py
python/paddle/utils/plotcurve.py
+2
-1
python/paddle/utils/predefined_net.py
python/paddle/utils/predefined_net.py
+3
-2
python/paddle/utils/preprocess_img.py
python/paddle/utils/preprocess_img.py
+7
-7
python/paddle/utils/preprocess_util.py
python/paddle/utils/preprocess_util.py
+5
-5
python/paddle/utils/show_pb.py
python/paddle/utils/show_pb.py
+5
-3
python/paddle/utils/torch2paddle.py
python/paddle/utils/torch2paddle.py
+2
-2
未找到文件。
python/paddle/utils/dump_config.py
浏览文件 @
812de6e8
...
...
@@ -37,9 +37,9 @@ if __name__ == '__main__':
assert
isinstance
(
conf
,
TrainerConfig_pb2
.
TrainerConfig
)
if
whole_conf
:
print
conf
print
(
conf
)
else
:
if
binary
:
sys
.
stdout
.
write
(
conf
.
model_config
.
SerializeToString
())
else
:
print
conf
.
model_config
print
(
conf
.
model_config
)
python/paddle/utils/image_multiproc.py
浏览文件 @
812de6e8
...
...
@@ -15,7 +15,8 @@
import
os
,
sys
import
numpy
as
np
from
PIL
import
Image
from
cStringIO
import
StringIO
import
six
from
six.moves
import
cStringIO
as
StringIO
import
multiprocessing
import
functools
import
itertools
...
...
@@ -187,7 +188,8 @@ class PILTransformer(ImageTransformer):
return
self
.
transform
(
im
)
def
job
(
is_img_string
,
transformer
,
(
data
,
label
)):
def
job
(
is_img_string
,
transformer
,
data_label_pack
):
(
data
,
label
)
=
data_label_pack
if
is_img_string
:
return
transformer
.
transform_from_string
(
data
),
label
else
:
...
...
@@ -208,7 +210,7 @@ class MultiProcessImageTransformer(object):
"""
Processing image with multi-process. If it is used in PyDataProvider,
the simple usage for CNN is as follows:
.. code-block:: python
def hool(settings, is_train, **kwargs):
...
...
@@ -229,7 +231,7 @@ class MultiProcessImageTransformer(object):
@provider(init_hook=hook, pool_size=20480)
def process(settings, file_list):
with open(file_list, 'r') as fdata:
for line in fdata:
for line in fdata:
data_dic = np.load(line.strip()) # load the data batch pickled by Pickle.
data = data_dic['data']
labels = data_dic['label']
...
...
@@ -249,10 +251,10 @@ class MultiProcessImageTransformer(object):
:type channel_swap: tuple or list
:param mean: the mean values of image, per-channel mean or element-wise mean.
:type mean: array, The dimension is 1 for per-channel mean.
The dimension is 3 for element-wise mean.
The dimension is 3 for element-wise mean.
:param is_train: training peroid or testing peroid.
:type is_train: bool.
:param is_color: the image is color or gray.
:param is_color: the image is color or gray.
:type is_color: bool.
:param is_img_string: The input can be the file name of image or image string.
:type is_img_string: bool.
...
...
@@ -273,4 +275,4 @@ class MultiProcessImageTransformer(object):
def
run
(
self
,
data
,
label
):
fun
=
functools
.
partial
(
job
,
self
.
is_img_string
,
self
.
transformer
)
return
self
.
pool
.
imap_unordered
(
fun
,
itertools
.
i
zip
(
data
,
label
),
chunksize
=
100
*
self
.
procnum
)
fun
,
six
.
moves
.
zip
(
data
,
label
),
chunksize
=
100
*
self
.
procnum
)
python/paddle/utils/image_util.py
浏览文件 @
812de6e8
...
...
@@ -14,7 +14,7 @@
import
numpy
as
np
from
PIL
import
Image
from
cStringIO
import
StringIO
from
six.moves
import
cStringIO
as
StringIO
def
resize_image
(
img
,
target_size
):
...
...
@@ -34,7 +34,7 @@ def flip(im):
"""
Return the flipped image.
Flip an image along the horizontal direction.
im: input image, (H x W x K) ndarrays
im: input image, (H x W x K) ndarrays
"""
if
len
(
im
.
shape
)
==
3
:
return
im
[:,
:,
::
-
1
]
...
...
@@ -132,7 +132,7 @@ def load_meta(meta_path, mean_img_size, crop_size, color=True):
def
load_image
(
img_path
,
is_color
=
True
):
"""
Load image and return.
Load image and return.
img_path: image path.
is_color: is color image or not.
"""
...
...
@@ -205,7 +205,7 @@ class ImageTransformer:
def
set_mean
(
self
,
mean
):
if
mean
is
not
None
:
# mean value, may be one value per channel
# mean value, may be one value per channel
if
mean
.
ndim
==
1
:
mean
=
mean
[:,
np
.
newaxis
,
np
.
newaxis
]
else
:
...
...
python/paddle/utils/make_model_diagram.py
浏览文件 @
812de6e8
...
...
@@ -15,6 +15,9 @@
# Generate dot diagram file for the given paddle model config
# The generated file can be viewed using Graphviz (http://graphviz.org)
from
__future__
import
print_function
import
six
import
sys
import
traceback
...
...
@@ -61,9 +64,9 @@ def make_diagram_from_proto(model_config, dot_file):
name2id
[
mem
.
link_name
])
return
s
print
>>
f
,
'digraph graphname {'
print
>>
f
,
'node [width=0.375,height=0.25];'
for
i
in
xrange
(
len
(
model_config
.
layers
)):
print
(
'digraph graphname {'
,
file
=
f
)
print
(
'node [width=0.375,height=0.25];'
,
file
=
f
)
for
i
in
six
.
moves
.
xrange
(
len
(
model_config
.
layers
)):
l
=
model_config
.
layers
[
i
]
name2id
[
l
.
name
]
=
i
...
...
@@ -71,12 +74,12 @@ def make_diagram_from_proto(model_config, dot_file):
for
sub_model
in
model_config
.
sub_models
:
if
sub_model
.
name
==
'root'
:
continue
print
>>
f
,
'subgraph cluster_%s {'
%
i
print
>>
f
,
'style=dashed;'
print
(
'subgraph cluster_%s {'
%
i
,
file
=
f
)
print
(
'style=dashed;'
,
file
=
f
)
label
=
'%s '
%
sub_model
.
name
if
sub_model
.
reversed
:
label
+=
'<=='
print
>>
f
,
'label = "%s";'
%
label
print
(
'label = "%s";'
%
label
,
file
=
f
)
i
+=
1
submodel_layers
.
add
(
sub_model
.
name
)
for
layer_name
in
sub_model
.
layer_names
:
...
...
@@ -84,37 +87,41 @@ def make_diagram_from_proto(model_config, dot_file):
lid
=
name2id
[
layer_name
]
layer_config
=
model_config
.
layers
[
lid
]
label
=
make_layer_label
(
layer_config
)
print
>>
f
,
'l%s [label="%s", shape=box];'
%
(
lid
,
label
)
print
>>
f
,
'}'
print
(
'l%s [label="%s", shape=box];'
%
(
lid
,
label
),
file
=
f
)
print
(
'}'
,
file
=
f
)
for
i
in
xrange
(
len
(
model_config
.
layers
)):
for
i
in
six
.
moves
.
xrange
(
len
(
model_config
.
layers
)):
l
=
model_config
.
layers
[
i
]
if
l
.
name
not
in
submodel_layers
:
label
=
make_layer_label
(
l
)
print
>>
f
,
'l%s [label="%s", shape=box];'
%
(
i
,
label
)
print
(
'l%s [label="%s", shape=box];'
%
(
i
,
label
),
file
=
f
)
for
sub_model
in
model_config
.
sub_models
:
if
sub_model
.
name
==
'root'
:
continue
for
link
in
sub_model
.
in_links
:
print
>>
f
,
make_link
(
link
)
print
(
make_link
(
link
),
file
=
f
)
for
link
in
sub_model
.
out_links
:
print
>>
f
,
make_link
(
link
)
print
(
make_link
(
link
),
file
=
f
)
for
mem
in
sub_model
.
memories
:
print
>>
f
,
make_mem
(
mem
)
print
(
make_mem
(
mem
),
file
=
f
)
for
i
in
xrange
(
len
(
model_config
.
layers
)):
for
i
in
six
.
moves
.
xrange
(
len
(
model_config
.
layers
)):
for
l
in
model_config
.
layers
[
i
].
inputs
:
print
>>
f
,
'l%s -> l%s [label="%s"];'
%
(
name2id
[
l
.
input_layer_name
],
i
,
l
.
input_parameter_name
)
print
(
'l%s -> l%s [label="%s"];'
%
(
name2id
[
l
.
input_layer_name
],
i
,
l
.
input_parameter_name
),
file
=
f
)
print
>>
f
,
'}'
print
(
'}'
,
file
=
f
)
f
.
close
()
def
usage
():
print
>>
sys
.
stderr
,
(
"Usage: python show_model_diagram.py"
+
" CONFIG_FILE DOT_FILE [config_str]"
)
print
(
(
"Usage: python show_model_diagram.py"
+
" CONFIG_FILE DOT_FILE [config_str]"
),
file
=
sys
.
stderr
)
exit
(
1
)
...
...
python/paddle/utils/merge_model.py
浏览文件 @
812de6e8
...
...
@@ -70,4 +70,4 @@ def merge_v2_model(net, param_file, output_file):
for
pname
in
param_names
:
params
.
serialize
(
pname
,
f
)
print
'Generate %s success!'
%
(
output_file
)
print
(
'Generate %s success!'
%
(
output_file
)
)
python/paddle/utils/plotcurve.py
浏览文件 @
812de6e8
...
...
@@ -44,6 +44,7 @@ To use this script to generate plot for AvgCost, error:
python plotcurve.py -i paddle.INFO -o figure.png AvgCost error
"""
import
six
import
sys
import
matplotlib
# the following line is added immediately after import matplotlib
...
...
@@ -91,7 +92,7 @@ def plot_paddle_curve(keys, inputfile, outputfile, format='png',
sys
.
stderr
.
write
(
"No data to plot. Exiting!
\n
"
)
return
m
=
len
(
keys
)
+
1
for
i
in
xrange
(
1
,
m
):
for
i
in
six
.
moves
.
xrange
(
1
,
m
):
pyplot
.
plot
(
x
[:,
0
],
x
[:,
i
],
...
...
python/paddle/utils/predefined_net.py
浏览文件 @
812de6e8
...
...
@@ -13,6 +13,7 @@
# limitations under the License.
import
numpy
as
np
import
six
import
os
from
paddle.trainer.config_parser
import
*
from
paddle.utils.preprocess_img
import
\
...
...
@@ -112,7 +113,7 @@ def simple_conv_net(data_conf, is_color=False):
num_classes: num of classes.
is_color: whether the input images are color.
"""
for
k
,
v
in
data_conf
.
iteritems
(
):
for
k
,
v
in
six
.
iteritems
(
data_conf
):
globals
()[
k
]
=
v
data_input
,
label_input
,
num_image_channels
=
\
image_data_layers
(
image_size
,
num_classes
,
is_color
,
is_predict
)
...
...
@@ -340,7 +341,7 @@ def small_vgg(data_conf, is_predict=False):
num_classes: num of classes.
is_color: whether the input images are color.
"""
for
k
,
v
in
data_conf
.
iteritems
(
):
for
k
,
v
in
six
.
iteritems
(
data_conf
):
globals
()[
k
]
=
v
vgg_conv_net
(
image_size
,
num_classes
,
num_layers
=
[
2
,
2
,
3
,
3
],
...
...
python/paddle/utils/preprocess_img.py
浏览文件 @
812de6e8
...
...
@@ -17,9 +17,9 @@ import os
import
random
import
numpy
as
np
import
PIL.Image
as
Image
import
StringIO
import
preprocess_util
from
image_util
import
crop_img
from
six.moves
import
cStringIO
as
StringIO
from
.
import
preprocess_util
from
.
image_util
import
crop_img
def
resize_image
(
img
,
target_size
):
...
...
@@ -52,7 +52,7 @@ class DiskImage:
def
read_image
(
self
):
if
self
.
img
is
None
:
print
"reading: "
+
self
.
path
print
(
"reading: "
+
self
.
path
)
image
=
resize_image
(
Image
.
open
(
self
.
path
),
self
.
target_size
)
self
.
img
=
image
...
...
@@ -69,7 +69,7 @@ class DiskImage:
convert the image into the paddle batch format.
"""
self
.
read_image
()
output
=
StringIO
.
StringIO
()
output
=
StringIO
()
self
.
img
.
save
(
output
,
"jpeg"
)
contents
=
output
.
getvalue
()
return
contents
...
...
@@ -127,7 +127,7 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater):
image_path
=
items
[
0
]
label_name
=
items
[
1
]
if
not
label_name
in
label_set
:
label_set
[
label_name
]
=
len
(
l
abel_set
.
keys
(
))
label_set
[
label_name
]
=
len
(
l
ist
(
label_set
.
keys
()
))
img
=
DiskImage
(
path
=
image_path
,
target_size
=
self
.
target_size
)
label
=
preprocess_util
.
Lablel
(
label
=
label_set
[
label_name
],
name
=
label_name
)
...
...
@@ -144,7 +144,7 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater):
return
create_dataset_from_list
(
path
)
label_set
=
preprocess_util
.
get_label_set_from_dir
(
path
)
data
=
[]
for
l_name
in
l
abel_set
.
keys
(
):
for
l_name
in
l
ist
(
label_set
.
keys
()
):
image_paths
=
preprocess_util
.
list_images
(
os
.
path
.
join
(
path
,
l_name
))
for
p
in
image_paths
:
...
...
python/paddle/utils/preprocess_util.py
浏览文件 @
812de6e8
...
...
@@ -14,7 +14,7 @@
import
os
import
math
import
cPickle
as
pickle
import
six.moves.
cPickle
as
pickle
import
random
import
collections
...
...
@@ -169,7 +169,7 @@ class Dataset:
random
.
shuffle
(
keyvalue_indices
[
k
])
num_data_per_key_batch
=
\
math
.
ceil
(
num_per_batch
/
float
(
len
(
keyvalue_indices
.
keys
(
))))
math
.
ceil
(
num_per_batch
/
float
(
len
(
list
(
keyvalue_indices
.
keys
()
))))
if
num_data_per_key_batch
<
2
:
raise
Exception
(
"The number of data in a batch is too small"
)
...
...
@@ -182,8 +182,8 @@ class Dataset:
end_idx
=
int
(
min
(
begin_idx
+
num_data_per_key_batch
,
len
(
keyvalue_indices
[
k
])))
print
"begin_idx, end_idx"
print
begin_idx
,
end_idx
print
(
"begin_idx, end_idx"
)
print
(
begin_idx
,
end_idx
)
for
idx
in
range
(
begin_idx
,
end_idx
):
permuted_data
.
append
(
self
.
data
[
keyvalue_indices
[
k
][
idx
]])
keyvalue_readpointer
[
k
]
=
end_idx
...
...
@@ -357,6 +357,6 @@ class DatasetCreater(object):
data_batcher
.
create_batches_and_list
(
self
.
output_path
,
self
.
train_list_name
,
self
.
test_list_name
,
self
.
label_set_name
)
self
.
num_classes
=
len
(
train_label_set
.
keys
(
))
self
.
num_classes
=
len
(
list
(
train_label_set
.
keys
()
))
self
.
create_meta_file
(
train_data
)
return
out_path
python/paddle/utils/show_pb.py
浏览文件 @
812de6e8
...
...
@@ -15,6 +15,8 @@
Show the content of proto buffer data file of PADDLE
"""
from
__future__
import
print_function
import
os
import
sys
from
google.protobuf.internal.decoder
import
_DecodeVarint
...
...
@@ -39,7 +41,7 @@ def read_proto(file, message):
def
usage
():
print
>>
sys
.
stderr
,
"Usage: python show_pb.py PROTO_DATA_FILE"
print
(
"Usage: python show_pb.py PROTO_DATA_FILE"
,
file
=
sys
.
stderr
)
exit
(
1
)
...
...
@@ -50,8 +52,8 @@ if __name__ == '__main__':
f
=
open
(
sys
.
argv
[
1
])
header
=
DataFormat
.
DataHeader
()
read_proto
(
f
,
header
)
print
header
print
(
header
)
sample
=
DataFormat
.
DataSample
()
while
read_proto
(
f
,
sample
):
print
sample
print
(
sample
)
python/paddle/utils/torch2paddle.py
浏览文件 @
812de6e8
...
...
@@ -24,7 +24,7 @@ import sys
import
struct
import
numpy
as
np
import
torchfile
import
cPickle
as
pickle
import
six.moves.
cPickle
as
pickle
import
argparse
...
...
@@ -48,7 +48,7 @@ def save_net_parameters(layers, params, output_path):
biases
=
params
[
i
*
2
+
1
]
weight_file
=
os
.
path
.
join
(
output_path
,
'_%s.w0'
%
layers
[
i
])
biases_file
=
os
.
path
.
join
(
output_path
,
'_%s.wbias'
%
layers
[
i
])
print
"Saving for layer %s."
%
layers
[
i
]
print
(
"Saving for layer %s."
%
layers
[
i
])
save_layer_parameters
(
weight_file
,
[
weight
])
save_layer_parameters
(
biases_file
,
biases
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录