Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
OPTHREE
Paddle
提交
c330af31
P
Paddle
项目概览
OPTHREE
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
c330af31
编写于
2月 23, 2017
作者:
D
dangqingqing
浏览文件
操作
浏览文件
下载
差异文件
Merge branch 'develop' of
https://github.com/PaddlePaddle/Paddle
into convert
上级
84b423a8
4cb84f33
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
74 addition
and
38 deletion
+74
-38
paddle/cuda/src/hl_warpctc_wrap.cc
paddle/cuda/src/hl_warpctc_wrap.cc
+10
-6
python/paddle/reader/decorator.py
python/paddle/reader/decorator.py
+48
-32
python/paddle/reader/tests/decorator_test.py
python/paddle/reader/tests/decorator_test.py
+16
-0
未找到文件。
paddle/cuda/src/hl_warpctc_wrap.cc
浏览文件 @
c330af31
...
...
@@ -54,22 +54,26 @@ DYNAMIC_LOAD_WARPCTC_WRAP(get_workspace_size)
#define WARPCTC_GET_VERSION dynload::get_warpctc_version
#define WARPCTC_GET_STATUS_STRING dynload::ctcGetStatusString
static
int
g_warpctcVersion
=
-
1
;
#ifndef PADDLE_TYPE_DOUBLE
#define WARPCTC_COMPUTE_LOSS dynload::compute_ctc_loss
#define WARPCTC_GET_WORKSPACE_SIZE dynload::get_workspace_size
#else
#define WARPCTC_LOG_FATAL \
LOG(FATAL) << "warp-ctc [version " << g_warpctcVersion \
<< "] Error: not support double precision."
#define WARPCTC_COMPUTE_LOSS(...) WARPCTC_LOG_FATAL(__VA_ARGS__)
#define WARPCTC_GET_WORKSPACE_SIZE(...) WARPCTC_LOG_FATAL(__VA_ARGS__)
hl_warpctc_status_t
fatal
(...)
{
LOG
(
FATAL
)
<<
"warp-ctc [version "
<<
g_warpctcVersion
<<
"] Error: not support double precision."
;
// both of get_warpctc_version() and get_workspace_size() return an ctcStatus
// type value
return
CTC_STATUS_EXECUTION_FAILED
;
}
#define WARPCTC_COMPUTE_LOSS fatal
#define WARPCTC_GET_WORKSPACE_SIZE fatal
#endif
/**
* Check build-in warp-ctc function using glog and it also
* support << operator for more details error info.
*/
static
int
g_warpctcVersion
=
-
1
;
#define CHECK_WARPCTC(warpctcStat) \
CHECK_EQ(CTC_STATUS_SUCCESS, warpctcStat) \
<< "warp-ctc [version " << g_warpctcVersion \
...
...
python/paddle/reader/decorator.py
浏览文件 @
c330af31
...
...
@@ -12,7 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
__all__
=
[
'buffered'
,
'compose'
,
'chain'
,
'shuffle'
,
'ComposeNotAligned'
]
__all__
=
[
'map_readers'
,
'buffered'
,
'compose'
,
'chain'
,
'shuffle'
,
'ComposeNotAligned'
]
from
Queue
import
Queue
from
threading
import
Thread
...
...
@@ -20,20 +23,38 @@ import itertools
import
random
def
map_readers
(
func
,
*
readers
):
"""
Creates a data reader that outputs return value of function using
output of each data readers as arguments.
:param func: function to use.
:param *readers: readers whose outputs will be used as arguments of func.
:returns: the created data reader.
"""
def
reader
():
rs
=
[]
for
r
in
readers
:
rs
.
append
(
r
())
for
e
in
itertools
.
imap
(
func
,
*
rs
):
yield
e
return
reader
def
shuffle
(
reader
,
buf_size
):
"""Creates a data reader whose data output is suffled.
"""
Creates a data reader whose data output is suffled.
Output from the iterator that created by original reader will be
buffered into shuffle buffer, and then shuffled. The size of shuffle buffer
is determined by argument buf_size.
Args:
reader: the original reader whose output will be
shuffled.
buf_size: shuffle buffer size.
:param reader: the original reader whose output will be shuffled.
:param buf_size: shuffle buffer size.
Returns:
the new reader whose output is shuffled.
:returns:the new reader whose output is shuffled.
"""
def
data_reader
():
...
...
@@ -55,8 +76,9 @@ def shuffle(reader, buf_size):
def
chain
(
*
readers
):
"""Creates a data reader whose output is the outputs of input data
readers chained together.
"""
Creates a data reader whose output is the outputs of input data
readers chained together.
If input readers output following data entries:
[0, 0, 0]
...
...
@@ -65,11 +87,8 @@ def chain(*readers):
The chained reader will output:
[0, 0, 0, 1, 1, 1, 2, 2, 2]
Args:
readers: input readers.
Returns:
the new data reader.
:param readers: input readers.
:returns: the new data reader.
"""
def
reader
():
...
...
@@ -88,25 +107,23 @@ class ComposeNotAligned(ValueError):
def
compose
(
*
readers
,
**
kwargs
):
"""Creates a data reader whose output is the combination of input readers.
"""
Creates a data reader whose output is the combination of input readers.
If input readers output following data entries:
(1, 2) 3 (4, 5)
The composed reader will output:
(1, 2, 3, 4, 5)
Args:
*readers: readers that will be composed together.
check_alignment: If True, will check if input readers are aligned
correctly. If False, will not check alignment and trailing outputs
will be discarded. Defaults to True.
:*readers: readers that will be composed together.
:check_alignment: if True, will check if input readers are aligned
correctly. If False, will not check alignment and trailing outputs
will be discarded. Defaults to True.
Returns:
the new data reader.
:returns: the new data reader.
Raises:
ComposeNotAligned: outputs of readers are not aligned.
Will not raise when check_alignment is set to False.
:raises ComposeNotAligned: outputs of readers are not aligned.
Will not raise when check_alignment is set to False.
"""
check_alignment
=
kwargs
.
pop
(
'check_alignment'
,
True
)
...
...
@@ -136,18 +153,17 @@ def compose(*readers, **kwargs):
def
buffered
(
reader
,
size
):
"""Creates a buffered data reader.
"""
Creates a buffered data reader.
The buffered data reader will read and save data entries into a
buffer. Reading from the buffered data reader will proceed as long
as the buffer is not empty.
Args:
reader: the data reader to read from.
size: max buffer size.
:param reader: the data reader to read from.
:param size: max buffer size.
Returns:
The buffered data reader.
:returns: the buffered data reader.
"""
class
EndSignal
():
...
...
python/paddle/reader/tests/decorator_test.py
浏览文件 @
c330af31
...
...
@@ -26,6 +26,22 @@ def reader_creator_10(dur):
return
reader
class
TestMap
(
unittest
.
TestCase
):
def
test_map
(
self
):
d
=
{
"h"
:
0
,
"i"
:
1
}
def
tokenize
(
x
):
return
d
[
x
]
def
read
():
yield
"h"
yield
"i"
r
=
paddle
.
reader
.
map_readers
(
tokenize
,
read
)
for
i
,
e
in
enumerate
(
r
()):
self
.
assertEqual
(
e
,
i
)
class
TestBuffered
(
unittest
.
TestCase
):
def
test_read
(
self
):
for
size
in
range
(
20
):
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录