Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
Paddle
提交
3fab4f65
P
Paddle
项目概览
PaddlePaddle
/
Paddle
大约 1 年 前同步成功
通知
2298
Star
20931
Fork
5422
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1423
列表
看板
标记
里程碑
合并请求
543
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1,423
Issue
1,423
列表
看板
标记
里程碑
合并请求
543
合并请求
543
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
3fab4f65
编写于
7月 03, 2018
作者:
F
fengjiayi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add EOFException to represent EOF in C++ reader
上级
61996d70
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
44 addition
and
17 deletion
+44
-17
paddle/fluid/framework/details/data_balance_op_handle.cc
paddle/fluid/framework/details/data_balance_op_handle.cc
+1
-1
paddle/fluid/framework/details/threaded_ssa_graph_executor.cc
...le/fluid/framework/details/threaded_ssa_graph_executor.cc
+18
-3
paddle/fluid/framework/details/threaded_ssa_graph_executor.h
paddle/fluid/framework/details/threaded_ssa_graph_executor.h
+1
-1
paddle/fluid/operators/read_op.cc
paddle/fluid/operators/read_op.cc
+1
-1
paddle/fluid/platform/enforce.h
paddle/fluid/platform/enforce.h
+15
-1
paddle/fluid/pybind/exception.cc
paddle/fluid/pybind/exception.cc
+3
-0
python/paddle/fluid/tests/unittests/test_data_balance.py
python/paddle/fluid/tests/unittests/test_data_balance.py
+2
-4
python/paddle/fluid/tests/unittests/test_multi_file_reader.py
...on/paddle/fluid/tests/unittests/test_multi_file_reader.py
+1
-2
python/paddle/fluid/tests/unittests/test_multi_pass_reader.py
...on/paddle/fluid/tests/unittests/test_multi_pass_reader.py
+1
-2
python/paddle/fluid/tests/unittests/test_recordio_reader.py
python/paddle/fluid/tests/unittests/test_recordio_reader.py
+1
-2
未找到文件。
paddle/fluid/framework/details/data_balance_op_handle.cc
浏览文件 @
3fab4f65
...
...
@@ -62,7 +62,7 @@ std::vector<std::array<int, 3>> DataBalanceOpHandle::GetBalancePlan(
}
if
(
total_size
<
device_num
)
{
// No enough data.
PADDLE_THROW
(
"There is no next data."
);
PADDLE_THROW
_EOF
(
);
}
std
::
sort
(
size_device_vec
.
begin
(),
size_device_vec
.
end
(),
[](
const
std
::
array
<
int
,
2
>
&
a
,
const
std
::
array
<
int
,
2
>
&
b
)
{
...
...
paddle/fluid/framework/details/threaded_ssa_graph_executor.cc
浏览文件 @
3fab4f65
...
...
@@ -98,9 +98,18 @@ FeedFetchList ThreadedSSAGraphExecutor::Run(
if
(
timeout
)
{
std
::
lock_guard
<
std
::
mutex
>
l
(
exception_mu_
);
if
(
exception_
)
{
auto
exp
=
*
exception_
;
exception_
.
reset
();
throw
exp
;
std
::
exception
*
exp
=
exception_
.
get
();
if
(
dynamic_cast
<
platform
::
EOFException
*>
(
exp
))
{
auto
e
=
*
static_cast
<
platform
::
EOFException
*>
(
exp
);
exception_
.
reset
();
throw
e
;
}
else
if
(
dynamic_cast
<
platform
::
EnforceNotMet
*>
(
exp
))
{
auto
e
=
*
static_cast
<
platform
::
EnforceNotMet
*>
(
exp
);
exception_
.
reset
();
throw
e
;
}
else
{
LOG
(
FATAL
)
<<
"Unknown exception."
;
}
}
else
{
continue
;
}
...
...
@@ -199,6 +208,12 @@ void ThreadedSSAGraphExecutor::RunOp(
running_ops_
--
;
ready_var_q
->
Extend
(
op
->
Outputs
());
VLOG
(
10
)
<<
op
<<
" "
<<
op
->
Name
()
<<
"Signal posted"
;
}
catch
(
platform
::
EOFException
ex
)
{
std
::
lock_guard
<
std
::
mutex
>
l
(
exception_mu_
);
// EOFException will not cover up existing EnforceNotMet.
if
(
exception_
.
get
()
==
nullptr
)
{
exception_
.
reset
(
new
platform
::
EOFException
(
ex
));
}
}
catch
(
platform
::
EnforceNotMet
ex
)
{
std
::
lock_guard
<
std
::
mutex
>
l
(
exception_mu_
);
exception_
.
reset
(
new
platform
::
EnforceNotMet
(
ex
));
...
...
paddle/fluid/framework/details/threaded_ssa_graph_executor.h
浏览文件 @
3fab4f65
...
...
@@ -57,7 +57,7 @@ class ThreadedSSAGraphExecutor : public SSAGraphExecutor {
std
::
vector
<
platform
::
Place
>
places_
;
platform
::
DeviceContextPool
fetch_ctxs_
;
std
::
mutex
exception_mu_
;
std
::
unique_ptr
<
platform
::
EnforceNotMet
>
exception_
;
std
::
unique_ptr
<
std
::
exception
>
exception_
;
std
::
atomic
<
int
>
running_ops_
;
void
InsertPendingOp
(
std
::
unordered_map
<
OpHandleBase
*
,
size_t
>
*
pending_ops
,
...
...
paddle/fluid/operators/read_op.cc
浏览文件 @
3fab4f65
...
...
@@ -68,7 +68,7 @@ class ReadOp : public framework::OperatorBase {
reader
->
ReadNext
(
&
ins
);
if
(
ins
.
empty
())
{
if
(
Attr
<
bool
>
(
"throw_eof_exp"
))
{
PADDLE_THROW
(
"There is no next data."
);
PADDLE_THROW
_EOF
(
);
}
else
{
ins
.
resize
(
out_arg_names
.
size
());
for
(
auto
&
tensor
:
ins
)
{
...
...
paddle/fluid/platform/enforce.h
浏览文件 @
3fab4f65
...
...
@@ -73,7 +73,7 @@ struct EnforceNotMet : public std::exception {
}
catch
(
const
std
::
exception
&
exp
)
{
std
::
ostringstream
sout
;
sout
<<
string
::
Sprintf
(
"
%s
at [%s:%d]"
,
exp
.
what
(),
f
,
l
)
<<
std
::
endl
;
sout
<<
string
::
Sprintf
(
"
'%s'
at [%s:%d]"
,
exp
.
what
(),
f
,
l
)
<<
std
::
endl
;
sout
<<
"PaddlePaddle Call Stacks: "
<<
std
::
endl
;
void
*
call_stack
[
TRACE_STACK_LIMIT
];
...
...
@@ -102,6 +102,15 @@ struct EnforceNotMet : public std::exception {
const
char
*
what
()
const
noexcept
{
return
err_str_
.
c_str
();
}
};
struct
EOFException
:
public
std
::
exception
{
std
::
string
err_str_
;
EOFException
(
const
char
*
err_msg
,
const
char
*
f
,
int
l
)
{
err_str_
=
string
::
Sprintf
(
"'%s' at [%s:%d]"
,
err_msg
,
f
,
l
);
}
const
char
*
what
()
const
noexcept
{
return
err_str_
.
c_str
();
}
};
// Because most enforce conditions would evaluate to true, we can use
// __builtin_expect to instruct the C++ compiler to generate code that
// always forces branch prediction of true.
...
...
@@ -242,6 +251,11 @@ inline void throw_on_error(T e) {
#define PADDLE_ENFORCE(...) ::paddle::platform::throw_on_error(__VA_ARGS__);
#endif
#define PADDLE_THROW_EOF() \
do { \
throw ::paddle::platform::EOFException("There is no next data.", __FILE__, \
__LINE__); \
} while (false)
/*
* Some enforce helpers here, usage:
* int a = 1;
...
...
paddle/fluid/pybind/exception.cc
浏览文件 @
3fab4f65
...
...
@@ -18,10 +18,13 @@ namespace paddle {
namespace
pybind
{
void
BindException
(
pybind11
::
module
*
m
)
{
static
pybind11
::
exception
<
platform
::
EOFException
>
eof
(
*
m
,
"EOFException"
);
static
pybind11
::
exception
<
platform
::
EnforceNotMet
>
exc
(
*
m
,
"EnforceNotMet"
);
pybind11
::
register_exception_translator
([](
std
::
exception_ptr
p
)
{
try
{
if
(
p
)
std
::
rethrow_exception
(
p
);
}
catch
(
const
platform
::
EOFException
&
e
)
{
eof
(
e
.
what
());
}
catch
(
const
platform
::
EnforceNotMet
&
e
)
{
exc
(
e
.
what
());
}
...
...
python/paddle/fluid/tests/unittests/test_data_balance.py
浏览文件 @
3fab4f65
...
...
@@ -118,8 +118,7 @@ class TestDataBalance(unittest.TestCase):
try
:
image_val
,
label_val
=
parallel_exe
.
run
(
fetch_list
,
return_numpy
=
True
)
except
fluid
.
core
.
EnforceNotMet
as
ex
:
self
.
assertIn
(
"There is no next data."
,
ex
.
message
)
except
fluid
.
core
.
EOFException
:
break
ins_num
=
image_val
.
shape
[
0
]
broadcasted_label
=
np
.
ones
(
...
...
@@ -162,8 +161,7 @@ class TestDataBalance(unittest.TestCase):
try
:
ins_tensor
,
label_tensor
=
parallel_exe
.
run
(
fetch_list
,
return_numpy
=
False
)
except
fluid
.
core
.
EnforceNotMet
as
ex
:
self
.
assertIn
(
"There is no next data."
,
ex
.
message
)
except
fluid
.
core
.
EOFException
:
break
ins_val
=
np
.
array
(
ins_tensor
)
...
...
python/paddle/fluid/tests/unittests/test_multi_file_reader.py
浏览文件 @
3fab4f65
...
...
@@ -64,8 +64,7 @@ class TestMultipleReader(unittest.TestCase):
while
True
:
try
:
img_val
,
=
exe
.
run
(
fetch_list
=
[
img
])
except
fluid
.
core
.
EnforceNotMet
as
ex
:
self
.
assertIn
(
"There is no next data."
,
ex
.
message
)
except
fluid
.
core
.
EOFException
:
break
batch_count
+=
1
self
.
assertLessEqual
(
img_val
.
shape
[
0
],
self
.
batch_size
)
...
...
python/paddle/fluid/tests/unittests/test_multi_pass_reader.py
浏览文件 @
3fab4f65
...
...
@@ -59,8 +59,7 @@ class TestMultipleReader(unittest.TestCase):
while
True
:
try
:
img_val
,
=
exe
.
run
(
fetch_list
=
[
img
])
except
fluid
.
core
.
EnforceNotMet
as
ex
:
self
.
assertIn
(
"There is no next data."
,
ex
.
message
)
except
fluid
.
core
.
EOFException
:
break
batch_count
+=
1
self
.
assertLessEqual
(
img_val
.
shape
[
0
],
self
.
batch_size
)
...
...
python/paddle/fluid/tests/unittests/test_recordio_reader.py
浏览文件 @
3fab4f65
...
...
@@ -68,8 +68,7 @@ class TestRecordIO(unittest.TestCase):
while
True
:
try
:
tmp
,
=
exe
.
run
(
fetch_list
=
[
avg_loss
])
except
fluid
.
core
.
EnforceNotMet
as
ex
:
self
.
assertIn
(
"There is no next data."
,
ex
.
message
)
except
fluid
.
core
.
EOFException
:
break
avg_loss_np
.
append
(
tmp
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录