Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
SummerGao.
Paddle
提交
d4d946db
P
Paddle
项目概览
SummerGao.
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d4d946db
编写于
6月 25, 2018
作者:
S
sneaxiy
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
update blocking queue
上级
67556e4a
变更
3
显示空白变更内容
内联
并排
Showing
3 changed file
with
27 addition
and
31 deletion
+27
-31
paddle/fluid/operators/reader/create_py_reader_op.cc
paddle/fluid/operators/reader/create_py_reader_op.cc
+6
-3
paddle/fluid/operators/reader/lod_tensor_blocking_queue.h
paddle/fluid/operators/reader/lod_tensor_blocking_queue.h
+15
-19
paddle/fluid/pybind/pybind.cc
paddle/fluid/pybind/pybind.cc
+6
-9
未找到文件。
paddle/fluid/operators/reader/create_py_reader_op.cc
浏览文件 @
d4d946db
...
...
@@ -28,7 +28,7 @@ class PyReader : public framework::ReaderBase {
void
ReadNext
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
{
bool
success
;
*
out
=
queue_
->
Dequeue
(
&
success
);
*
out
=
queue_
->
Pop
(
&
success
);
if
(
!
success
)
out
->
clear
();
}
...
...
@@ -45,6 +45,10 @@ class CreatePyReaderOp : public framework::OperatorBase {
private:
void
RunImpl
(
const
framework
::
Scope
&
scope
,
const
platform
::
Place
&
dev_place
)
const
override
{
auto
*
out
=
scope
.
FindVar
(
Output
(
"Out"
))
->
template
GetMutable
<
framework
::
ReaderHolder
>();
if
(
out
->
Get
()
!=
nullptr
)
return
;
const
std
::
string
&
queue_name
=
Input
(
"blocking_queue"
);
auto
*
queue_holder_var
=
scope
.
FindVar
(
queue_name
);
PADDLE_ENFORCE
(
...
...
@@ -53,8 +57,7 @@ class CreatePyReaderOp : public framework::OperatorBase {
queue_name
);
auto
*
queue_holder
=
queue_holder_var
->
template
GetMutable
<
LoDTensorBlockingQueueHolder
>();
auto
*
out
=
scope
.
FindVar
(
Output
(
"Out"
))
->
template
GetMutable
<
framework
::
ReaderHolder
>();
out
->
Reset
(
new
PyReader
(
queue_holder
->
GetQueue
()));
}
};
...
...
paddle/fluid/operators/reader/lod_tensor_blocking_queue.h
浏览文件 @
d4d946db
...
...
@@ -34,36 +34,33 @@ class LoDTensorBlockingQueue {
private:
LoDTensorBlockingQueue
(
size_t
capacity
,
const
std
::
vector
<
framework
::
DDim
>&
dims
)
:
dims_
(
dims
)
{
queue_
.
reset
(
new
BlockingQueue
<
std
::
vector
<
framework
::
LoDTensor
>>
(
capacity
));
}
:
queue_
(
capacity
),
dims_
(
dims
)
{}
public:
bool
Enqueue
(
const
std
::
vector
<
framework
::
LoDTensor
>&
lod_tensor_vec
)
{
bool
Push
(
const
std
::
vector
<
framework
::
LoDTensor
>&
lod_tensor_vec
)
{
CheckDims
(
lod_tensor_vec
);
return
queue_
->
Send
(
lod_tensor_vec
);
return
queue_
.
Send
(
lod_tensor_vec
);
}
bool
Enqueue
(
std
::
vector
<
framework
::
LoDTensor
>&&
lod_tensor_vec
)
{
bool
Push
(
std
::
vector
<
framework
::
LoDTensor
>&&
lod_tensor_vec
)
{
CheckDims
(
lod_tensor_vec
);
return
queue_
->
Send
(
std
::
move
(
lod_tensor_vec
));
return
queue_
.
Send
(
std
::
move
(
lod_tensor_vec
));
}
std
::
vector
<
framework
::
LoDTensor
>
Dequeue
(
bool
*
ok
=
nullptr
)
{
std
::
vector
<
framework
::
LoDTensor
>
Pop
(
bool
*
ok
=
nullptr
)
{
std
::
vector
<
framework
::
LoDTensor
>
lod_tensor_vec
;
bool
success
=
queue_
->
Receive
(
&
lod_tensor_vec
);
bool
success
=
queue_
.
Receive
(
&
lod_tensor_vec
);
if
(
ok
!=
nullptr
)
*
ok
=
success
;
return
lod_tensor_vec
;
}
inline
size_t
Cap
()
const
{
return
queue_
->
Cap
();
}
inline
size_t
Cap
()
const
{
return
queue_
.
Cap
();
}
inline
size_t
Size
()
const
{
return
queue_
->
Size
();
}
inline
size_t
Size
()
const
{
return
queue_
.
Size
();
}
inline
void
Close
()
{
return
queue_
->
Close
();
}
inline
void
Close
()
{
return
queue_
.
Close
();
}
inline
bool
IsClosed
()
const
{
return
queue_
->
IsClosed
();
}
inline
bool
IsClosed
()
const
{
return
queue_
.
IsClosed
();
}
private:
void
CheckDims
(
const
std
::
vector
<
framework
::
LoDTensor
>&
lod_tensor_vec
)
{
...
...
@@ -71,15 +68,16 @@ class LoDTensorBlockingQueue {
"Expect input size is %d but found %s"
,
dims_
.
size
(),
lod_tensor_vec
.
size
());
for
(
size_t
i
=
0
;
i
<
dims_
.
size
();
++
i
)
{
const
auto
&
in_dims
=
lod_tensor_vec
[
i
].
dims
();
const
auto
&
in_dims
=
framework
::
slice_ddim
(
lod_tensor_vec
[
i
].
dims
(),
1
,
lod_tensor_vec
[
i
].
dims
().
size
());
const
auto
&
expect_dims
=
framework
::
slice_ddim
(
dims_
[
i
],
1
,
dims_
[
i
].
size
());
PADDLE_ENFORCE
(
in_dims
==
expect_dims
,
"Dims of the %d-th input tensor do
es
not match"
,
i
);
"Dims of the %d-th input tensor do not match"
,
i
);
}
}
std
::
unique_ptr
<
BlockingQueue
<
std
::
vector
<
framework
::
LoDTensor
>
>>
queue_
;
BlockingQueue
<
std
::
vector
<
framework
::
LoDTensor
>>
queue_
;
std
::
vector
<
framework
::
DDim
>
dims_
;
};
...
...
@@ -92,8 +90,6 @@ class LoDTensorBlockingQueueHolder {
queue_
.
reset
(
new
LoDTensorBlockingQueue
(
capacity
,
dims
));
}
inline
std
::
shared_ptr
<
LoDTensorBlockingQueue
>
GetQueue
()
{
return
queue_
;
}
inline
const
std
::
shared_ptr
<
LoDTensorBlockingQueue
>&
GetQueue
()
const
{
return
queue_
;
}
...
...
paddle/fluid/pybind/pybind.cc
浏览文件 @
d4d946db
...
...
@@ -303,19 +303,16 @@ All parameter, weight, gradient are variables in Paddle.
using
LoDTensorBlockingQueueHolder
=
::
paddle
::
operators
::
reader
::
LoDTensorBlockingQueueHolder
;
py
::
class_
<
LoDTensorBlockingQueue
>
(
m
,
"LoDTensorBlockingQueue"
,
""
)
.
def
(
"
enqueue
"
,
.
def
(
"
push
"
,
[](
LoDTensorBlockingQueue
&
self
,
const
std
::
vector
<
framework
::
LoDTensor
>
&
lod_tensor_vec
)
{
pybind11
::
gil_scoped_release
release
;
return
self
.
Enqueue
(
lod_tensor_vec
);
return
self
.
Push
(
lod_tensor_vec
);
})
.
def
(
"size"
,
[](
const
LoDTensorBlockingQueue
&
self
)
{
return
self
.
Size
();
})
.
def
(
"capacity"
,
[](
const
LoDTensorBlockingQueue
&
self
)
{
return
self
.
Cap
();
})
.
def
(
"close"
,
[](
LoDTensorBlockingQueue
&
self
)
{
return
self
.
Close
();
})
.
def
(
"is_closed"
,
[](
const
LoDTensorBlockingQueue
&
self
)
{
return
self
.
IsClosed
();
});
.
def
(
"size"
,
&
LoDTensorBlockingQueue
::
Size
)
.
def
(
"capacity"
,
&
LoDTensorBlockingQueue
::
Cap
)
.
def
(
"close"
,
&
LoDTensorBlockingQueue
::
Close
)
.
def
(
"is_closed"
,
&
LoDTensorBlockingQueue
::
IsClosed
);
m
.
def
(
"init_lod_tensor_blocking_queue"
,
[](
Variable
&
var
,
size_t
capacity
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录