Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
DeepSpeech
提交
2e5e9b8c
D
DeepSpeech
项目概览
PaddlePaddle
/
DeepSpeech
12 个月 前同步成功
通知
205
Star
8425
Fork
1598
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
245
列表
看板
标记
里程碑
合并请求
3
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
D
DeepSpeech
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
245
Issue
245
列表
看板
标记
里程碑
合并请求
3
合并请求
3
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
2e5e9b8c
编写于
10月 11, 2017
作者:
X
Xinghai Sun
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Turn on rnn_use_batch of Paddle for accelartion.
Improve xmap_reader_mp by adding a flush thread.
上级
8e5c2eb9
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
37 addition
and
12 deletion
+37
-12
data_utils/utility.py
data_utils/utility.py
+27
-9
infer.py
infer.py
+3
-1
test.py
test.py
+3
-1
tools/tune.py
tools/tune.py
+3
-1
train.py
train.py
+1
-0
未找到文件。
data_utils/utility.py
浏览文件 @
2e5e9b8c
...
...
@@ -9,6 +9,7 @@ import os
import
tarfile
import
time
from
Queue
import
Queue
from
threading
import
Thread
from
multiprocessing
import
Process
,
Manager
from
paddle.v2.dataset.common
import
md5file
...
...
@@ -100,7 +101,8 @@ def xmap_readers_mp(mapper, reader, process_num, buffer_size, order=False):
in_queue
.
put
((
order_id
,
sample
))
in_queue
.
put
(
end_flag
)
# define a worker to handle samples from in_queue by mapper and put results to out_queue
# define a worker to handle samples from in_queue by mapper and put results
# to out_queue
def
handle_worker
(
in_queue
,
out_queue
,
mapper
):
sample
=
in_queue
.
get
()
while
not
isinstance
(
sample
,
XmapEndSignal
):
...
...
@@ -109,7 +111,8 @@ def xmap_readers_mp(mapper, reader, process_num, buffer_size, order=False):
in_queue
.
put
(
end_flag
)
out_queue
.
put
(
end_flag
)
# define a worker to handle samples from in_queue by mapper and put results to out_queue with order
# define a worker to handle samples from in_queue by mapper and put results
# to out_queue with order
def
order_handle_worker
(
in_queue
,
out_queue
,
mapper
,
out_order
):
ins
=
in_queue
.
get
()
while
not
isinstance
(
ins
,
XmapEndSignal
):
...
...
@@ -123,6 +126,18 @@ def xmap_readers_mp(mapper, reader, process_num, buffer_size, order=False):
in_queue
.
put
(
end_flag
)
out_queue
.
put
(
end_flag
)
# define a thread worker to flush samples from Manager.Queue to Queue
# for acceleration
def
flush_worker
(
in_queue
,
out_queue
):
finish
=
0
while
finish
<
process_num
:
sample
=
in_queue
.
get
()
if
isinstance
(
sample
,
XmapEndSignal
):
finish
+=
1
else
:
out_queue
.
put
(
sample
)
out_queue
.
put
(
end_flag
)
def
xreader
():
# prepare shared memory
manager
=
Manager
()
...
...
@@ -147,13 +162,16 @@ def xmap_readers_mp(mapper, reader, process_num, buffer_size, order=False):
w
.
daemon
=
True
w
.
start
()
# start a thread to read data from slow Manager.Queue
flush_queue
=
Queue
(
buffer_size
)
t
=
Thread
(
target
=
flush_worker
,
args
=
(
out_queue
,
flush_queue
))
t
.
daemon
=
True
t
.
start
()
# get results
finish
=
0
while
finish
<
process_num
:
sample
=
out_queue
.
get
()
if
isinstance
(
sample
,
XmapEndSignal
):
finish
+=
1
else
:
yield
sample
sample
=
flush_queue
.
get
()
while
not
isinstance
(
sample
,
XmapEndSignal
):
yield
sample
sample
=
flush_queue
.
get
()
return
xreader
infer.py
浏览文件 @
2e5e9b8c
...
...
@@ -116,7 +116,9 @@ def infer():
def
main
():
print_arguments
(
args
)
paddle
.
init
(
use_gpu
=
args
.
use_gpu
,
trainer_count
=
args
.
trainer_count
)
paddle
.
init
(
use_gpu
=
args
.
use_gpu
,
rnn_use_batch
=
True
,
trainer_count
=
args
.
trainer_count
)
infer
()
...
...
test.py
浏览文件 @
2e5e9b8c
...
...
@@ -119,7 +119,9 @@ def evaluate():
def
main
():
print_arguments
(
args
)
paddle
.
init
(
use_gpu
=
args
.
use_gpu
,
trainer_count
=
args
.
trainer_count
)
paddle
.
init
(
use_gpu
=
args
.
use_gpu
,
rnn_use_batch
=
True
,
trainer_count
=
args
.
trainer_count
)
evaluate
()
...
...
tools/tune.py
浏览文件 @
2e5e9b8c
...
...
@@ -217,7 +217,9 @@ def tune():
def
main
():
print_arguments
(
args
)
paddle
.
init
(
use_gpu
=
args
.
use_gpu
,
trainer_count
=
args
.
trainer_count
)
paddle
.
init
(
use_gpu
=
args
.
use_gpu
,
rnn_use_batch
=
True
,
trainer_count
=
args
.
trainer_count
)
tune
()
...
...
train.py
浏览文件 @
2e5e9b8c
...
...
@@ -119,6 +119,7 @@ def train():
def
main
():
print_arguments
(
args
)
paddle
.
init
(
use_gpu
=
args
.
use_gpu
,
rnn_use_batch
=
True
,
trainer_count
=
args
.
trainer_count
,
log_clipping
=
True
)
train
()
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录