训练几次迭代后出错
Created by: zhengzhe97
Traceback (most recent call last): File "tools/train.py", line 323, in main() File "tools/train.py", line 233, in main outs = exe.run(compiled_train_prog, fetch_list=train_values) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py", line 783, in run six.reraise(*sys.exc_info()) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/six.py", line 693, in reraise raise value File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py", line 778, in run use_program_cache=use_program_cache) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py", line 843, in _run_impl return_numpy=return_numpy) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py", line 677, in _run_parallel tensors = exe.run(fetch_var_names)._move_to_list() paddle.fluid.core_avx.EnforceNotMet:
C++ Call Stacks (More useful to developers):
0 std::string paddle::platform::GetTraceBackString<char const*>(char const*&&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) 2 paddle::platform::CUDADeviceContext::Wait() const 3 paddle::framework::details::ScopeBufferedSSAGraphExecutor::DropLocalExeScopes() 4 paddle::framework::details::ScopeBufferedSSAGraphExecutor::Run(std::vector<std::string, std::allocatorstd::string > const&) 5 paddle::framework::ParallelExecutor::Run(std::vector<std::string, std::allocatorstd::string > const&)
Error Message Summary:
FatalError: cudaStreamSynchronize raises error: an illegal memory access was encountered, errono: 77: an illegal memory access was encountered at (/paddle/paddle/fluid/platform/device_context.cc:331)
terminate called after throwing an instance of 'paddle::platform::EnforceNotMet' what():
C++ Call Stacks (More useful to developers):
0 std::string paddle::platform::GetTraceBackString<char const*>(char const*&&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) 2 paddle::framework::details::OpHandleBase::~OpHandleBase() 3 paddle::framework::details::ComputationOpHandle::~ComputationOpHandle() 4 paddle::framework::ir::Node::~Node() 5 paddle::framework::ir::Node::~Node() 6 std::_Rb_tree<paddle::framework::ir::Node*, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 7 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 8 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 9 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 10 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 11 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 12 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 13 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 14 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 15 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 16 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 17 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 18 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 19 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 20 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >) 21 std::_Rb_tree<paddle::framework::ir::Node, std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > >, std::_Select1st<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >, std::lesspaddle::framework::ir::Node*, std::allocator<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > > >::_M_erase(std::_Rb_tree_node<std::pair<paddle::framework::ir::Node* const, std::unique_ptr<paddle::framework::ir::Node, std::default_deletepaddle::framework::ir::Node > > >*) 22 paddle::framework::ir::Graph::~Graph() 23 paddle::framework::ir::Graph::~Graph() 24 std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release()
Error Message Summary:
Error: An error occurred here. There is no accurate error hint for this error yet. We are continuously in the process of increasing hint for this kind of error check. It would be helpful if you could inform us of how this conversion went by opening a github issue. And we will resolve it with high priority.
- New issue link: https://github.com/PaddlePaddle/Paddle/issues/new
- Recommended issue content: all error stack information: an illegal memory access was encountered at (/paddle/paddle/fluid/framework/details/op_handle_base.cc:39)
W0320 22:43:11.314461 19896 init.cc:209] Warning: PaddlePaddle catches a failure signal, it may not work properly W0320 22:43:11.314502 19896 init.cc:211] You could check whether you killed PaddlePaddle thread/process accidentally or report the case to PaddlePaddle W0320 22:43:11.314505 19896 init.cc:214] The detail failure signal is:
W0320 22:43:11.314517 19896 init.cc:217] *** Aborted at 1584715391 (unix time) try "date -d @1584715391" if you are using GNU date *** W0320 22:43:11.316820 19896 init.cc:217] PC: @ 0x0 (unknown) W0320 22:43:11.316900 19896 init.cc:217] *** SIGABRT (@0x3e800004db8) received by PID 19896 (TID 0x7f1f8f745700) from PID 19896; stack trace: *** W0320 22:43:11.318758 19896 init.cc:217] @ 0x7f1f8f329390 (unknown) W0320 22:43:11.320505 19896 init.cc:217] @ 0x7f1f8ef83428 gsignal W0320 22:43:11.322242 19896 init.cc:217] @ 0x7f1f8ef8502a abort W0320 22:43:11.323382 19896 init.cc:217] @ 0x7f1f4feee84a __gnu_cxx::__verbose_terminate_handler() W0320 22:43:11.324288 19896 init.cc:217] @ 0x7f1f4feecf47 __cxxabiv1::__terminate() W0320 22:43:11.325177 19896 init.cc:217] @ 0x7f1f4feec3a5 __cxa_call_terminate W0320 22:43:11.326150 19896 init.cc:217] @ 0x7f1f4feecbd8 __gxx_personality_v0 W0320 22:43:11.327016 19896 init.cc:217] @ 0x7f1f501dfaab _Unwind_RaiseException_Phase2 W0320 22:43:11.327883 19896 init.cc:217] @ 0x7f1f501dff49 _Unwind_Resume W0320 22:43:11.331857 19896 init.cc:217] @ 0x7f1f2bb9bcdc paddle::framework::details::OpHandleBase::~OpHandleBase() W0320 22:43:11.335081 19896 init.cc:217] @ 0x7f1f2bb9ea6f paddle::framework::details::ComputationOpHandle::~ComputationOpHandle() W0320 22:43:11.338526 19896 init.cc:217] @ 0x7f1f29596ee9 paddle::framework::ir::Node::~Node() W0320 22:43:11.343547 19896 init.cc:217] @ 0x7f1f29597091 paddle::framework::ir::Node::~Node() W0320 22:43:11.348611 19896 init.cc:217] @ 0x7f1f29581172 std::_Rb_tree<>::_M_erase() W0320 22:43:11.353679 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.358695 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.363716 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.368736 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.373762 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.378774 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.383782 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.388830 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.393842 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.398867 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.403884 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.408905 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.413939 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.418946 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.423964 19896 init.cc:217] @ 0x7f1f2958115f std::_Rb_tree<>::_M_erase() W0320 22:43:11.426046 19896 init.cc:217] @ 0x7f1f2bdffe0e paddle::framework::ir::Graph::~Graph() W0320 22:43:11.430371 19896 init.cc:217] @ 0x7f1f2bdffea1 paddle::framework::ir::Graph::~Graph() W0320 22:43:11.432696 19896 init.cc:217] @ 0x7f1f292c8659 std::_Sp_counted_base<>::_M_release() Aborted (core dumped)