Created by: kuke

Need to improve the profiler to support multi-gpu mode. The log:

-----------  Configuration Arguments -----------
batch_size: 32
device: GPU
feature_lst: data/feature.lst
first_batches_to_skip: 1
hidden_dim: 1024
label_lst: data/label.lst
learning_rate: 0.002
max_batch_num: 10
mean_var: data/global_mean_var_search26kHr
minimum_batch_size: 1
parallel: True
print_train_acc: False
proj_dim: 512
sorted_key: total
stacked_num: 5
------------------------------------------------
F0224 03:39:43.402420 20900 threadpool.h:96] The exception is thrown inside the thread pool. You should use RunAndGetException to handle the exception.
The default exception handler is LOG(FATAL).invalid resource handle at [/paddle_work/Paddle/paddle/fluid/platform/profiler.cc:59]
PaddlePaddle Call Stacks:
0       0x7f8b5bb5ba7cp paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 572
1       0x7f8b5bc09803p paddle::platform::Event::Event(paddle::platform::EventKind, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned int, paddle::platform::DeviceContext const*) + 979
2       0x7f8b5bc0b2aap paddle::platform::PushEvent(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, paddle::platform::DeviceContext const*) + 314
3       0x7f8b5bc0b89dp paddle::platform::RecordEvent::RecordEvent(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, paddle::platform::DeviceContext const*) + 93
4       0x7f8b5bbfebc8p paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool) + 1336
5       0x7f8b5c4a7f33p std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > >, std::__future_base::_Result_base::_Deleter>, std::_Bind_simple<std::reference_wrapper<std::future<std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > > paddle::framework::ThreadPool::RunAndGetException<paddle::operators::ParallelDoOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const::{lambda()#1}>(paddle::operators::ParallelDoOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const::{lambda()#1})::{lambda()#1}> ()>, std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<paddle::platform::EnforceNotMet> > > >::_M_invoke(std::_Any_data const&) + 99
6       0x7f8b5c4a4a1ep std::__future_base::_State_baseV2::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*) + 46
7       0x7f8b95a05a99p
8       0x7f8b5c4a5062p std::__future_base::_State_baseV2::_M_set_result(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>, bool) + 146
9       0x7f8b5c4a5296p std::__future_base::_Task_state<std::future<std::unique_ptr<paddle::platform::EnforceNotMet, std::default_delete<std::unique_ptr> > > paddle::framework::ThreadPool::RunAndGetException<paddle::operators::ParallelDoOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const::{lambda()#1}>(paddle::operators::ParallelDoOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const::{lambda()#1})::{lambda()#1}, std::allocator<int>, std::default_delete<std::unique_ptr> ()>::_M_run() + 86
10      0x7f8b5c6a86b4p paddle::framework::ThreadPool::TaskLoop() + 1012
11      0x7f8b89d98c80p
12      0x7f8b959fe6bap
13      0x7f8b9573482dp clone + 109
*** Check failure stack trace: ***
    @     0x7f8b5c7814dd  google::LogMessage::Fail()
    @     0x7f8b5c783828  google::LogMessage::SendToLog()
    @     0x7f8b5c780feb  google::LogMessage::Flush()
    @     0x7f8b5c7846fe  google::LogMessageFatal::~LogMessageFatal()
    @     0x7f8b5c4a5cd7  std::_Function_handler<>::_M_invoke()
    @     0x7f8b5c4a4a1e  std::__future_base::_State_baseV2::_M_do_set()
    @     0x7f8b95a05a99  __pthread_once_slow
    @     0x7f8b5c4a5062  std::__future_base::_State_baseV2::_M_set_result()
    @     0x7f8b5c4a5121  std::__future_base::_Deferred_state<>::_M_complete_async()
    @     0x7f8b5c4ad762  paddle::operators::ParallelDoOp::RunImpl()
    @     0x7f8b5bbfec8e  paddle::framework::Executor::Run()
    @     0x7f8b5bb78433  _ZZN8pybind1112cpp_function10initializeIZNS0_C4IvN6paddle9framework8ExecutorEIRKNS4_11ProgramDescEPNS4_5ScopeEibbEINS_4nameENS_9is_methodENS_7siblingEEEEMT0_FT_DpT1_EDpRKT2_EUlPS5_S8_SA_ibbE_vISO_S8_SA_ibbEISB_SC_SD_EEEvOSF_PFSE_SH_ESN_ENUlRNS_6detail13function_callEE1_4_FUNESV_
    @     0x7f8b5bb76174  pybind11::cpp_function::dispatcher()
    @           0x4cada2  PyEval_EvalFrameEx
    @           0x4c2765  PyEval_EvalCodeEx
    @           0x4ca099  PyEval_EvalFrameEx
    @           0x4c2765  PyEval_EvalCodeEx
    @           0x4ca8d1  PyEval_EvalFrameEx
    @           0x4c2765  PyEval_EvalCodeEx
    @           0x4c2509  PyEval_EvalCode
    @           0x4f1def  (unknown)
    @           0x4ec652  PyRun_FileExFlags
    @           0x4eae31  PyRun_SimpleFileExFlags
    @           0x49e14a  Py_Main
    @     0x7f8b9564e830  __libc_start_main
    @           0x49d9d9  _start
    @              (nil)  (unknown)
Aborted