Created by: Rayarrow94

GPU训练报错：

Traceback (most recent call last): File "train.py", line 734, in train(args) File "train.py", line 685, in train train_exe.run(fetch_list=[]) File "/home/disk2/chenjian11/tools/python36/lib/python3.6/site-packages/paddle/fluid/parallel_executor.py", line 303, in run self.executor.run(fetch_list, fetch_var_name) paddle.fluid.core.EnforceNotMet: Invoke operator read error. Python Callstacks: File "/home/disk2/chenjian11/tools/python36/lib/python3.6/site-packages/paddle/fluid/framework.py", line 1317, in append_op attrs=kwargs.get("attrs", None)) File "/home/disk2/chenjian11/tools/python36/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 56, in append_op return self.main_program.current_block().append_op(args, kwargs) File "/home/disk2/chenjian11/tools/python36/lib/python3.6/site-packages/paddle/fluid/layers/io.py", line 1075, in read_file type='read', inputs={'Reader': [reader]}, outputs={'Out': out}) File "train.py", line 287, in create_model label, inputmask_query_ids, inputmask_gs_feed_title, inputmask_rec_query = fluid.layers.read_file(pyreader) File "train.py", line 565, in train cost_ad = create_model(pyreader_name='train_reader', ernie_config=ernie_config) File "train.py", line 734, in train(args) C++ Callstacks: Enforce failed. Expected ins.size() == out_arg_names.size(), but received ins.size():1 != out_arg_names.size():45. at [/paddle/paddle/fluid/operators/reader/read_op.cc:107] PaddlePaddle Call Stacks: 0 0x7f6d2382437dp void paddle::platform::EnforceNotMet::Initstd::string(std::string, char const, int) + 365 1 0x7f6d238246c7p paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const, int) + 87 2 0x7f6d244a57e5p paddle::operators::ReadOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 3237 3 0x7f6d252ff175p paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 341 4 0x7f6d25176169p 5 0x7f6d2516ecedp 6 0x7f6d2516e635p paddle::framework::details::OpHandleBase::RunAndRecordEvent(std::function<void ()> const&) + 805 7 0x7f6d25175dfcp paddle::framework::details::ComputationOpHandle::RunImpl() + 124 8 0x7f6d2516f556p paddle::framework::details::OpHandleBase::Run(bool) + 118 9 0x7f6d2510776dp 10 0x7f6d2510816dp paddle::framework::details::ThreadedSSAGraphExecutor::RunOp(std::shared_ptr<paddle::framework::BlockingQueuepaddle::framework::details::VarHandleBase* > const&, paddle::framework::details::OpHandleBase) + 1309 11 0x7f6d25109de4p paddle::framework::details::ThreadedSSAGraphExecutor::Run(std::vector<std::string, std::allocatorstd::string > const&) + 2324 12 0x7f6d25100e6ap paddle::framework::details::ScopeBufferedSSAGraphExecutor::Run(std::vector<std::string, std::allocatorstd::string > const&) + 394 13 0x7f6d2395f1f2p paddle::framework::ParallelExecutor::Run(std::vector<std::string, std::allocatorstd::string > const&, std::string const&) + 562 14 0x7f6d23813fcep 15 0x7f6d2384f1eep 16 0x7f6d8f1db9fdp _PyCFunction_FastCallDict + 557 17 0x7f6d8f273f18p 18 0x7f6d8f278b56p _PyEval_EvalFrameDefault + 16774 19 0x7f6d8f273daep 20 0x7f6d8f2740c2p 21 0x7f6d8f2785a8p _PyEval_EvalFrameDefault + 15320 22 0x7f6d8f273440p 23 0x7f6d8f274364p 24 0x7f6d8f278b56p _PyEval_EvalFrameDefault + 16774 25 0x7f6d8f273daep 26 0x7f6d8f2743ddp PyEval_EvalCodeEx + 109 27 0x7f6d8f27442bp PyEval_EvalCode + 59 28 0x7f6d8f2af2b2p PyRun_FileExFlags + 178 29 0x7f6d8f2af417p PyRun_SimpleFileExFlags + 231 30 0x7f6d8f2cb51dp Py_Main + 3757 31 0x400bbcp main + 364 32 0x7f6d8e45bbd5p __libc_start_main + 245 33 0x400c7dp

terminate called without an active exception

* Aborted at 1564063111 (unix time) try "date -d @1564063111" if you are using GNU date *

PC: @ 0x0 (unknown)

* SIGABRT (@0x1fc00004672) received by PID 18034 (TID 0x7f67f293e700) from PID 18034; stack trace: *

@ 0x7f6d8ef01160 (unknown) @ 0x7f6d8e46f3f7 __GI_raise @ 0x7f6d8e4707d8 __GI_abort @ 0x7f6d8247cc65 __gnu_cxx::__verbose_terminate_handler() @ 0x7f6d8247ae06 __cxxabiv1::__terminate() @ 0x7f6d8247ae33 std::terminate() @ 0x7f6d8247aa87 __gxx_personality_v0 @ 0x7f6d8294d964 _Unwind_ForcedUnwind_Phase2 @ 0x7f6d8294dcb4 _Unwind_ForcedUnwind @ 0x7f6d8eeffc30 __GI___pthread_unwind @ 0x7f6d8eefa565 __pthread_exit @ 0x7f6d8f2c71ff PyThread_exit_thread @ 0x7f6d8f27306a PyEval_RestoreThread @ 0x7f6d2383ca29 pybind11::gil_scoped_release::~gil_scoped_release() @ 0x7f6d238060e3 ZZN8pybind1112cpp_function10initializeIZN6paddle6pybindL18pybind11_init_coreERNS_6moduleEEUlRNS2_9operators6reader22LoDTensorBlockingQueueERKSt6vectorINS2_9framework9LoDTensorESaISC_EEE60_bIS9_SG_EINS_4nameENS_9is_methodENS_7siblingEEEEvOT_PFT0_DpT1_EDpRKT2_ENUlRNS_6detail13function_callEE1_4_FUNESY @ 0x7f6d2384f1ee pybind11::cpp_function::dispatcher() @ 0x7f6d8f1db9fd _PyCFunction_FastCallDict @ 0x7f6d8f273f18 call_function @ 0x7f6d8f278b56 _PyEval_EvalFrameDefault @ 0x7f6d8f273dae _PyEval_EvalCodeWithName @ 0x7f6d8f2743dd PyEval_EvalCodeEx @ 0x7f6d8f1b4893 function_call @ 0x7f6d8f18241a PyObject_Call @ 0x7f6d8f2784af _PyEval_EvalFrameDefault @ 0x7f6d8f273440 _PyFunction_FastCall @ 0x7f6d8f274364 call_function @ 0x7f6d8f278b56 _PyEval_EvalFrameDefault @ 0x7f6d8f273440 _PyFunction_FastCall @ 0x7f6d8f274364 call_function @ 0x7f6d8f278b56 _PyEval_EvalFrameDefault @ 0x7f6d8f273440 _PyFunction_FastCall @ 0x7f6d8f27cb56 _PyFunction_FastCallDict Aborted