GPU单机多卡ParallelExecutor模式出现如下异常
Created by: xymyeah
terminate called after throwing an instance of 'paddle::platform::EnforceNotMet'
what(): an illegal memory access was encountered at [/paddle/paddle/fluid/framework/details/op_handle_base.cc:37]
PaddlePaddle Call Stacks:
0 0x7fce59686456p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 486
1 0x7fce5a107bdfp paddle::framework::details::OpHandleBase::~OpHandleBase() + 463
2 0x7fce5a104b01p paddle::framework::details::FetchOpHandle::~FetchOpHandle() + 17
3 0x7fce5a10236ep std::vector<std::unique_ptr<paddle::framework::details::FetchOpHandle, std::default_delete<paddle::framework::details::FetchOpHandle> >, std::all
ocator<std::unique_ptr<paddle::framework::details::FetchOpHandle, std::default_delete<paddle::framework::details::FetchOpHandle> > > >::~vector() + 46
4 0x7fce5a100e48p paddle::framework::details::ThreadedSSAGraphExecutor::Run(std::vector<std::string, std::allocator<std::string> > const&) + 3656
5 0x7fce5972ca8dp paddle::framework::ParallelExecutor::Run(std::vector<std::string, std::allocator<std::string> > const&, std::string const&) + 365
6 0x7fce596b9026p _ZZN8pybind1112cpp_function10initializeIZNS0_C1IvN6paddle9framework16ParallelExecutorEJRKSt6vectorISsSaISsEERKSsEJNS_4nameENS_9is_methodENS_7sibl
ingEEEEMT0_FT_DpT1_EDpRKT2_EUlPS5_SA_SC_E_vJSQ_SA_SC_EJSD_SE_SF_EEEvOSH_PFSG_SJ_ESP_ENUlRNS_6detail13function_callEE1_4_FUNESX_ + 294
7 0x7fce59697564p pybind11::cpp_function::dispatcher(_object*, _object*, _object*) + 2596
8 0x7fceea46e631p PyEval_EvalFrameEx + 24497
9 0x7fceea46fbcep PyEval_EvalCodeEx + 2190
10 0x7fceea46e20ap PyEval_EvalFrameEx + 23434
11 0x7fceea46fbcep PyEval_EvalCodeEx + 2190
12 0x7fceea46e20ap PyEval_EvalFrameEx + 23434
13 0x7fceea46fbcep PyEval_EvalCodeEx + 2190
14 0x7fceea46fce2p PyEval_EvalCode + 50
15 0x7fceea48f9e0p PyRun_FileExFlags + 176
16 0x7fceea48fbbfp PyRun_SimpleFileExFlags + 239
17 0x7fceea4a5454p Py_Main + 3188
18 0x7fcee9759cddp __libc_start_main + 253
19 0x400649p
*** Aborted at 1524487301 (unix time) try "date -d @1524487301" if you are using GNU date ***
PC: @ 0x0 (unknown)
*** SIGABRT (@0x803) received by PID 2051 (TID 0x7fceeab68700) from PID 2051; stack trace: ***
@ 0x7fceea168500 (unknown)
@ 0x7fcee976d8a5 __GI_raise
@ 0x7fcee976f085 __GI_abort
@ 0x7fcee0329a5d __gnu_cxx::__verbose_terminate_handler()
@ 0x7fcee0327be6 (unknown)
@ 0x7fcee0326b79 (unknown)
@ 0x7fcee03275d1 __gxx_personality_v0
@ 0x7fcee49e7323 (unknown)
@ 0x7fcee49e76ad _Unwind_RaiseException
@ 0x7fcee0327d01 __cxa_throw
@ 0x7fce5a107bfd paddle::framework::details::OpHandleBase::~OpHandleBase()
@ 0x7fce5a104b01 paddle::framework::details::FetchOpHandle::~FetchOpHandle()
@ 0x7fce5a10236e std::vector<>::~vector()
@ 0x7fce5a100e48 paddle::framework::details::ThreadedSSAGraphExecutor::Run()
@ 0x7fce5972ca8d paddle::framework::ParallelExecutor::Run()
@ 0x7fce596b9026 _ZZN8pybind1112cpp_function10initializeIZNS0_C1IvN6paddle9framework16ParallelExecutorEIRKSt6vectorISsSaISsEERKSsEINS_4nameENS_9is_methodENS_7sib
lingEEEEMT0_FT_DpT1_EDpRKT2_EUlPS5_SA_SC_E_vISQ_SA_SC_EISD_SE_SF_EEEvOSH_PFSG_SJ_ESP_ENUlRNS_6detail13function_callEE1_4_FUNESX_
@ 0x7fce59697564 pybind11::cpp_function::dispatcher()