Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
极客李华
Paddle
提交
3fab4f65
P
Paddle
项目概览
极客李华
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
3fab4f65
编写于
7月 03, 2018
作者:
F
fengjiayi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add EOFException to represent EOF in C++ reader
上级
61996d70
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
44 addition
and
17 deletion
+44
-17
paddle/fluid/framework/details/data_balance_op_handle.cc
paddle/fluid/framework/details/data_balance_op_handle.cc
+1
-1
paddle/fluid/framework/details/threaded_ssa_graph_executor.cc
...le/fluid/framework/details/threaded_ssa_graph_executor.cc
+18
-3
paddle/fluid/framework/details/threaded_ssa_graph_executor.h
paddle/fluid/framework/details/threaded_ssa_graph_executor.h
+1
-1
paddle/fluid/operators/read_op.cc
paddle/fluid/operators/read_op.cc
+1
-1
paddle/fluid/platform/enforce.h
paddle/fluid/platform/enforce.h
+15
-1
paddle/fluid/pybind/exception.cc
paddle/fluid/pybind/exception.cc
+3
-0
python/paddle/fluid/tests/unittests/test_data_balance.py
python/paddle/fluid/tests/unittests/test_data_balance.py
+2
-4
python/paddle/fluid/tests/unittests/test_multi_file_reader.py
...on/paddle/fluid/tests/unittests/test_multi_file_reader.py
+1
-2
python/paddle/fluid/tests/unittests/test_multi_pass_reader.py
...on/paddle/fluid/tests/unittests/test_multi_pass_reader.py
+1
-2
python/paddle/fluid/tests/unittests/test_recordio_reader.py
python/paddle/fluid/tests/unittests/test_recordio_reader.py
+1
-2
未找到文件。
paddle/fluid/framework/details/data_balance_op_handle.cc
浏览文件 @
3fab4f65
...
...
@@ -62,7 +62,7 @@ std::vector<std::array<int, 3>> DataBalanceOpHandle::GetBalancePlan(
}
if
(
total_size
<
device_num
)
{
// No enough data.
PADDLE_THROW
(
"There is no next data."
);
PADDLE_THROW
_EOF
(
);
}
std
::
sort
(
size_device_vec
.
begin
(),
size_device_vec
.
end
(),
[](
const
std
::
array
<
int
,
2
>
&
a
,
const
std
::
array
<
int
,
2
>
&
b
)
{
...
...
paddle/fluid/framework/details/threaded_ssa_graph_executor.cc
浏览文件 @
3fab4f65
...
...
@@ -98,9 +98,18 @@ FeedFetchList ThreadedSSAGraphExecutor::Run(
if
(
timeout
)
{
std
::
lock_guard
<
std
::
mutex
>
l
(
exception_mu_
);
if
(
exception_
)
{
auto
exp
=
*
exception_
;
exception_
.
reset
();
throw
exp
;
std
::
exception
*
exp
=
exception_
.
get
();
if
(
dynamic_cast
<
platform
::
EOFException
*>
(
exp
))
{
auto
e
=
*
static_cast
<
platform
::
EOFException
*>
(
exp
);
exception_
.
reset
();
throw
e
;
}
else
if
(
dynamic_cast
<
platform
::
EnforceNotMet
*>
(
exp
))
{
auto
e
=
*
static_cast
<
platform
::
EnforceNotMet
*>
(
exp
);
exception_
.
reset
();
throw
e
;
}
else
{
LOG
(
FATAL
)
<<
"Unknown exception."
;
}
}
else
{
continue
;
}
...
...
@@ -199,6 +208,12 @@ void ThreadedSSAGraphExecutor::RunOp(
running_ops_
--
;
ready_var_q
->
Extend
(
op
->
Outputs
());
VLOG
(
10
)
<<
op
<<
" "
<<
op
->
Name
()
<<
"Signal posted"
;
}
catch
(
platform
::
EOFException
ex
)
{
std
::
lock_guard
<
std
::
mutex
>
l
(
exception_mu_
);
// EOFException will not cover up existing EnforceNotMet.
if
(
exception_
.
get
()
==
nullptr
)
{
exception_
.
reset
(
new
platform
::
EOFException
(
ex
));
}
}
catch
(
platform
::
EnforceNotMet
ex
)
{
std
::
lock_guard
<
std
::
mutex
>
l
(
exception_mu_
);
exception_
.
reset
(
new
platform
::
EnforceNotMet
(
ex
));
...
...
paddle/fluid/framework/details/threaded_ssa_graph_executor.h
浏览文件 @
3fab4f65
...
...
@@ -57,7 +57,7 @@ class ThreadedSSAGraphExecutor : public SSAGraphExecutor {
std
::
vector
<
platform
::
Place
>
places_
;
platform
::
DeviceContextPool
fetch_ctxs_
;
std
::
mutex
exception_mu_
;
std
::
unique_ptr
<
platform
::
EnforceNotMet
>
exception_
;
std
::
unique_ptr
<
std
::
exception
>
exception_
;
std
::
atomic
<
int
>
running_ops_
;
void
InsertPendingOp
(
std
::
unordered_map
<
OpHandleBase
*
,
size_t
>
*
pending_ops
,
...
...
paddle/fluid/operators/read_op.cc
浏览文件 @
3fab4f65
...
...
@@ -68,7 +68,7 @@ class ReadOp : public framework::OperatorBase {
reader
->
ReadNext
(
&
ins
);
if
(
ins
.
empty
())
{
if
(
Attr
<
bool
>
(
"throw_eof_exp"
))
{
PADDLE_THROW
(
"There is no next data."
);
PADDLE_THROW
_EOF
(
);
}
else
{
ins
.
resize
(
out_arg_names
.
size
());
for
(
auto
&
tensor
:
ins
)
{
...
...
paddle/fluid/platform/enforce.h
浏览文件 @
3fab4f65
...
...
@@ -73,7 +73,7 @@ struct EnforceNotMet : public std::exception {
}
catch
(
const
std
::
exception
&
exp
)
{
std
::
ostringstream
sout
;
sout
<<
string
::
Sprintf
(
"
%s
at [%s:%d]"
,
exp
.
what
(),
f
,
l
)
<<
std
::
endl
;
sout
<<
string
::
Sprintf
(
"
'%s'
at [%s:%d]"
,
exp
.
what
(),
f
,
l
)
<<
std
::
endl
;
sout
<<
"PaddlePaddle Call Stacks: "
<<
std
::
endl
;
void
*
call_stack
[
TRACE_STACK_LIMIT
];
...
...
@@ -102,6 +102,15 @@ struct EnforceNotMet : public std::exception {
const
char
*
what
()
const
noexcept
{
return
err_str_
.
c_str
();
}
};
struct
EOFException
:
public
std
::
exception
{
std
::
string
err_str_
;
EOFException
(
const
char
*
err_msg
,
const
char
*
f
,
int
l
)
{
err_str_
=
string
::
Sprintf
(
"'%s' at [%s:%d]"
,
err_msg
,
f
,
l
);
}
const
char
*
what
()
const
noexcept
{
return
err_str_
.
c_str
();
}
};
// Because most enforce conditions would evaluate to true, we can use
// __builtin_expect to instruct the C++ compiler to generate code that
// always forces branch prediction of true.
...
...
@@ -242,6 +251,11 @@ inline void throw_on_error(T e) {
#define PADDLE_ENFORCE(...) ::paddle::platform::throw_on_error(__VA_ARGS__);
#endif
#define PADDLE_THROW_EOF() \
do { \
throw ::paddle::platform::EOFException("There is no next data.", __FILE__, \
__LINE__); \
} while (false)
/*
* Some enforce helpers here, usage:
* int a = 1;
...
...
paddle/fluid/pybind/exception.cc
浏览文件 @
3fab4f65
...
...
@@ -18,10 +18,13 @@ namespace paddle {
namespace
pybind
{
void
BindException
(
pybind11
::
module
*
m
)
{
static
pybind11
::
exception
<
platform
::
EOFException
>
eof
(
*
m
,
"EOFException"
);
static
pybind11
::
exception
<
platform
::
EnforceNotMet
>
exc
(
*
m
,
"EnforceNotMet"
);
pybind11
::
register_exception_translator
([](
std
::
exception_ptr
p
)
{
try
{
if
(
p
)
std
::
rethrow_exception
(
p
);
}
catch
(
const
platform
::
EOFException
&
e
)
{
eof
(
e
.
what
());
}
catch
(
const
platform
::
EnforceNotMet
&
e
)
{
exc
(
e
.
what
());
}
...
...
python/paddle/fluid/tests/unittests/test_data_balance.py
浏览文件 @
3fab4f65
...
...
@@ -118,8 +118,7 @@ class TestDataBalance(unittest.TestCase):
try
:
image_val
,
label_val
=
parallel_exe
.
run
(
fetch_list
,
return_numpy
=
True
)
except
fluid
.
core
.
EnforceNotMet
as
ex
:
self
.
assertIn
(
"There is no next data."
,
ex
.
message
)
except
fluid
.
core
.
EOFException
:
break
ins_num
=
image_val
.
shape
[
0
]
broadcasted_label
=
np
.
ones
(
...
...
@@ -162,8 +161,7 @@ class TestDataBalance(unittest.TestCase):
try
:
ins_tensor
,
label_tensor
=
parallel_exe
.
run
(
fetch_list
,
return_numpy
=
False
)
except
fluid
.
core
.
EnforceNotMet
as
ex
:
self
.
assertIn
(
"There is no next data."
,
ex
.
message
)
except
fluid
.
core
.
EOFException
:
break
ins_val
=
np
.
array
(
ins_tensor
)
...
...
python/paddle/fluid/tests/unittests/test_multi_file_reader.py
浏览文件 @
3fab4f65
...
...
@@ -64,8 +64,7 @@ class TestMultipleReader(unittest.TestCase):
while
True
:
try
:
img_val
,
=
exe
.
run
(
fetch_list
=
[
img
])
except
fluid
.
core
.
EnforceNotMet
as
ex
:
self
.
assertIn
(
"There is no next data."
,
ex
.
message
)
except
fluid
.
core
.
EOFException
:
break
batch_count
+=
1
self
.
assertLessEqual
(
img_val
.
shape
[
0
],
self
.
batch_size
)
...
...
python/paddle/fluid/tests/unittests/test_multi_pass_reader.py
浏览文件 @
3fab4f65
...
...
@@ -59,8 +59,7 @@ class TestMultipleReader(unittest.TestCase):
while
True
:
try
:
img_val
,
=
exe
.
run
(
fetch_list
=
[
img
])
except
fluid
.
core
.
EnforceNotMet
as
ex
:
self
.
assertIn
(
"There is no next data."
,
ex
.
message
)
except
fluid
.
core
.
EOFException
:
break
batch_count
+=
1
self
.
assertLessEqual
(
img_val
.
shape
[
0
],
self
.
batch_size
)
...
...
python/paddle/fluid/tests/unittests/test_recordio_reader.py
浏览文件 @
3fab4f65
...
...
@@ -68,8 +68,7 @@ class TestRecordIO(unittest.TestCase):
while
True
:
try
:
tmp
,
=
exe
.
run
(
fetch_list
=
[
avg_loss
])
except
fluid
.
core
.
EnforceNotMet
as
ex
:
self
.
assertIn
(
"There is no next data."
,
ex
.
message
)
except
fluid
.
core
.
EOFException
:
break
avg_loss_np
.
append
(
tmp
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录