Profiling failed in multi-gpu mode
Created by: kuke
Need to improve the profiler to support multi-gpu mode. The log:
----------- Configuration Arguments -----------
batch_size: 32
device: GPU
feature_lst: data/feature.lst
first_batches_to_skip: 1
hidden_dim: 1024
label_lst: data/label.lst
learning_rate: 0.002
max_batch_num: 10
mean_var: data/global_mean_var_search26kHr
minimum_batch_size: 1
parallel: True
print_train_acc: False
proj_dim: 512
sorted_key: total
stacked_num: 5
------------------------------------------------
F0224 03:39:43.402420 20900 threadpool.h:96] The exception is thrown inside the thread pool. You should use RunAndGetException to handle the exception.
The default exception handler is LOG(FATAL).invalid resource handle at [/paddle_work/Paddle/paddle/fluid/platform/profiler.cc:59]
PaddlePaddle Call Stacks:
0 0x7f8b5bb5ba7cp paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 572
1 0x7f8b5bc09803p paddle::platform::Event::Event(paddle::platform::EventKind, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned int, paddle::platform::DeviceContext const*) + 979
2 0x7f8b5bc0b2aap paddle::platform::PushEvent(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, paddle::platform::DeviceContext const*) + 314
3 0x7f8b5bc0b89dp paddle::platform::RecordEvent::RecordEvent(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, paddle::platform::DeviceContext const*) + 93
4 0x7f8b5bbfebc8p paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool) + 1336
5 0x7f8b5c4a7f33p std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > >, std::__future_base::_Result_base::_Deleter>, std::_Bind_simple<std::reference_wrapper<std::future<std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > > paddle::framework::ThreadPool::RunAndGetException<paddle::operators::ParallelDoOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const::{lambda()#1}>(paddle::operators::ParallelDoOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const::{lambda()#1})::{lambda()#1}> ()>, std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > > >::_M_invoke(std::_Any_data const&) + 99
6 0x7f8b5c4a4a1ep std::__future_base::_State_baseV2::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*) + 46
7 0x7f8b95a05a99p
8 0x7f8b5c4a5062p std::__future_base::_State_baseV2::_M_set_result(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>, bool) + 146
9 0x7f8b5c4a5296p std::__future_base::_Task_state<std::future<std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<std::unique_ptr> > > paddle::framework::ThreadPool::RunAndGetException<paddle::operators::ParallelDoOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const::{lambda()#1}>(paddle::operators::ParallelDoOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const::{lambda()#1})::{lambda()#1}, std::allocator<int>, std::default_delete<std::unique_ptr> ()>::_M_run() + 86
10 0x7f8b5c6a86b4p paddle::framework::ThreadPool::TaskLoop() + 1012
11 0x7f8b89d98c80p
12 0x7f8b959fe6bap
13 0x7f8b9573482dp clone + 109
*** Check failure stack trace: ***
@ 0x7f8b5c7814dd google::LogMessage::Fail()
@ 0x7f8b5c783828 google::LogMessage::SendToLog()
@ 0x7f8b5c780feb google::LogMessage::Flush()
@ 0x7f8b5c7846fe google::LogMessageFatal::~LogMessageFatal()
@ 0x7f8b5c4a5cd7 std::_Function_handler<>::_M_invoke()
@ 0x7f8b5c4a4a1e std::__future_base::_State_baseV2::_M_do_set()
@ 0x7f8b95a05a99 __pthread_once_slow
@ 0x7f8b5c4a5062 std::__future_base::_State_baseV2::_M_set_result()
@ 0x7f8b5c4a5121 std::__future_base::_Deferred_state<>::_M_complete_async()
@ 0x7f8b5c4ad762 paddle::operators::ParallelDoOp::RunImpl()
@ 0x7f8b5bbfec8e paddle::framework::Executor::Run()
@ 0x7f8b5bb78433 _ZZN8pybind1112cpp_function10initializeIZNS0_C4IvN6paddle9framework8ExecutorEIRKNS4_11ProgramDescEPNS4_5ScopeEibbEINS_4nameENS_9is_methodENS_7siblingEEEEMT0_FT_DpT1_EDpRKT2_EUlPS5_S8_SA_ibbE_vISO_S8_SA_ibbEISB_SC_SD_EEEvOSF_PFSE_SH_ESN_ENUlRNS_6detail13function_callEE1_4_FUNESV_
@ 0x7f8b5bb76174 pybind11::cpp_function::dispatcher()
@ 0x4cada2 PyEval_EvalFrameEx
@ 0x4c2765 PyEval_EvalCodeEx
@ 0x4ca099 PyEval_EvalFrameEx
@ 0x4c2765 PyEval_EvalCodeEx
@ 0x4ca8d1 PyEval_EvalFrameEx
@ 0x4c2765 PyEval_EvalCodeEx
@ 0x4c2509 PyEval_EvalCode
@ 0x4f1def (unknown)
@ 0x4ec652 PyRun_FileExFlags
@ 0x4eae31 PyRun_SimpleFileExFlags
@ 0x49e14a Py_Main
@ 0x7f8b9564e830 __libc_start_main
@ 0x49d9d9 _start
@ (nil) (unknown)
Aborted