Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
models
提交
dd008f15
M
models
项目概览
PaddlePaddle
/
models
大约 1 年 前同步成功
通知
222
Star
6828
Fork
2962
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
602
列表
看板
标记
里程碑
合并请求
255
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
models
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
602
Issue
602
列表
看板
标记
里程碑
合并请求
255
合并请求
255
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
dd008f15
编写于
2月 05, 2018
作者:
Y
yangyaming
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Make batch assembling parallel.
上级
f12deac8
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
53 addition
and
32 deletion
+53
-32
fluid/DeepASR/data_utils/data_reader.py
fluid/DeepASR/data_utils/data_reader.py
+53
-32
未找到文件。
fluid/DeepASR/data_utils/data_reader.py
浏览文件 @
dd008f15
...
...
@@ -5,13 +5,14 @@ from __future__ import division
from
__future__
import
print_function
import
random
import
numpy
as
np
import
struct
import
Queue
import
time
import
numpy
as
np
from
threading
import
Thread
from
multiprocessing
import
Manager
,
Process
import
data_utils.augmentor.trans_mean_variance_norm
as
trans_mean_variance_norm
import
data_utils.augmentor.trans_add_delta
as
trans_add_delta
from
multiprocessing
import
Manager
,
Process
from
threading
import
Thread
import
time
class
SampleInfo
(
object
):
...
...
@@ -127,6 +128,8 @@ class DataReader(object):
cached.
sample_info_buffer_size (int): Buffer size to indicate the maximum
sample information cached.
batch_buffer_size (int): Buffer size to indicate the maximum batch
cached.
shuffle_block_num (int): Block number indicating the minimum unit to do
shuffle.
random_seed (int): Random seed.
...
...
@@ -141,7 +144,8 @@ class DataReader(object):
drop_frame_len
=
256
,
process_num
=
10
,
sample_buffer_size
=
1024
,
sample_info_buffer_size
=
10000
,
sample_info_buffer_size
=
1024
,
batch_buffer_size
=
1024
,
shuffle_block_num
=
1
,
random_seed
=
0
):
self
.
_feature_file_list
=
feature_file_list
...
...
@@ -158,6 +162,7 @@ class DataReader(object):
self
.
_manager
=
Manager
()
self
.
_sample_buffer_size
=
sample_buffer_size
self
.
_sample_info_buffer_size
=
sample_info_buffer_size
self
.
_batch_buffer_size
=
batch_buffer_size
self
.
_process_num
=
process_num
def
generate_bucket_list
(
self
,
is_shuffle
):
...
...
@@ -197,7 +202,7 @@ class DataReader(object):
sample_queue
=
self
.
_manager
.
Queue
(
self
.
_sample_buffer_size
)
self
.
_order_id
=
0
def
ordered_feeding_
worker
(
sample_info_queue
):
def
ordered_feeding_
task
(
sample_info_queue
):
for
sample_info_bucket
in
self
.
_bucket_list
:
sample_info_list
=
sample_info_bucket
.
generate_sample_info_list
(
)
...
...
@@ -210,12 +215,11 @@ class DataReader(object):
sample_info_queue
.
put
(
EpochEndSignal
())
feeding_thread
=
Thread
(
target
=
ordered_feeding_
worker
,
args
=
(
sample_info_queue
,
))
target
=
ordered_feeding_
task
,
args
=
(
sample_info_queue
,
))
feeding_thread
.
daemon
=
True
feeding_thread
.
start
()
def
ordered_processing_worker
(
sample_info_queue
,
sample_queue
,
out_order
):
def
ordered_processing_task
(
sample_info_queue
,
sample_queue
,
out_order
):
def
read_bytes
(
fpath
,
start
,
size
):
f
=
open
(
fpath
,
'r'
)
f
.
seek
(
start
,
0
)
...
...
@@ -273,7 +277,7 @@ class DataReader(object):
args
=
(
sample_info_queue
,
sample_queue
,
out_order
)
workers
=
[
Process
(
target
=
ordered_processing_
worker
,
args
=
args
)
target
=
ordered_processing_
task
,
args
=
args
)
for
_
in
xrange
(
self
.
_process_num
)
]
...
...
@@ -295,13 +299,27 @@ class DataReader(object):
w
.
join
()
def
batch_iterator
(
self
,
batch_size
,
minimum_batch_size
):
batch_samples
=
[]
lod
=
[
0
]
# check whether need parallel here
for
sample
in
self
.
_sample_generator
():
batch_samples
.
append
(
sample
)
lod
.
append
(
lod
[
-
1
]
+
sample
[
0
].
shape
[
0
])
if
len
(
batch_samples
)
==
batch_size
:
def
batch_assembling_task
(
sample_generator
,
batch_queue
):
batch_samples
=
[]
lod
=
[
0
]
for
sample
in
sample_generator
():
batch_samples
.
append
(
sample
)
lod
.
append
(
lod
[
-
1
]
+
sample
[
0
].
shape
[
0
])
if
len
(
batch_samples
)
==
batch_size
:
batch_feature
=
np
.
zeros
(
(
lod
[
-
1
],
self
.
_frame_dim
),
dtype
=
"float32"
)
batch_label
=
np
.
zeros
((
lod
[
-
1
],
1
),
dtype
=
"int64"
)
start
=
0
for
sample
in
batch_samples
:
frame_num
=
sample
[
0
].
shape
[
0
]
batch_feature
[
start
:
start
+
frame_num
,
:]
=
sample
[
0
]
batch_label
[
start
:
start
+
frame_num
,
:]
=
sample
[
1
]
start
+=
frame_num
batch_queue
.
put
((
batch_feature
,
batch_label
,
lod
))
batch_samples
=
[]
lod
=
[
0
]
if
len
(
batch_samples
)
>=
minimum_batch_size
:
batch_feature
=
np
.
zeros
(
(
lod
[
-
1
],
self
.
_frame_dim
),
dtype
=
"float32"
)
batch_label
=
np
.
zeros
((
lod
[
-
1
],
1
),
dtype
=
"int64"
)
...
...
@@ -311,18 +329,21 @@ class DataReader(object):
batch_feature
[
start
:
start
+
frame_num
,
:]
=
sample
[
0
]
batch_label
[
start
:
start
+
frame_num
,
:]
=
sample
[
1
]
start
+=
frame_num
yield
(
batch_feature
,
batch_label
,
lod
)
batch_samples
=
[]
lod
=
[
0
]
if
len
(
batch_samples
)
>=
minimum_batch_size
:
batch_feature
=
np
.
zeros
(
(
lod
[
-
1
],
self
.
_frame_dim
),
dtype
=
"float32"
)
batch_label
=
np
.
zeros
((
lod
[
-
1
],
1
),
dtype
=
"int64"
)
start
=
0
for
sample
in
batch_samples
:
frame_num
=
sample
[
0
].
shape
[
0
]
batch_feature
[
start
:
start
+
frame_num
,
:]
=
sample
[
0
]
batch_label
[
start
:
start
+
frame_num
,
:]
=
sample
[
1
]
start
+=
frame_num
yield
(
batch_feature
,
batch_label
,
lod
)
batch_queue
.
put
((
batch_feature
,
batch_label
,
lod
))
batch_queue
.
put
(
EpochEndSignal
())
batch_queue
=
Queue
.
Queue
(
self
.
_batch_buffer_size
)
assembling_thread
=
Thread
(
target
=
batch_assembling_task
,
args
=
(
self
.
_sample_generator
,
batch_queue
))
assembling_thread
.
daemon
=
True
assembling_thread
.
start
()
batch_data
=
batch_queue
.
get
()
while
not
isinstance
(
batch_data
,
EpochEndSignal
):
yield
batch_data
batch_data
=
batch_queue
.
get
()
assembling_thread
.
join
()
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录