Created by: ivychen0515

参考官网链接 http://www.paddlepaddle.org/paddle/ModelBase#%E8%87%AA%E7%84%B6%E8%AF%AD%E8%A8%80%E5%A4%84%E7%90%86%0A%E6%9C%BA%E5%99%A8%E7%BF%BB%E8%AF%91-Transformer 训练平台 Centos 6.3; CUDA9.0; PaddlePaddle 1.3.0 报错信息如下 Traceback (most recent call last): File "train.py", line 772, in train(args) File "train.py", line 691, in train token_num, predict, pyreader) File "train.py", line 544, in train_loop feed=feed_dict_list) File "/home/work/anaconda3/envs/pytorch/lib/python3.6/site-packages/paddle/fluid/parallel_executor.py", line 303, in run self.executor.run(fetch_list, fetch_var_name) paddle.fluid.core.EnforceNotMet: Invoke operator lookup_table error. Python Callstacks: File "/home/work/anaconda3/envs/pytorch/lib/python3.6/site-packages/paddle/fluid/framework.py", line 1317, in append_op attrs=kwargs.get("attrs", None)) File "/home/work/anaconda3/envs/pytorch/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 56, in append_op return self.main_program.current_block().append_op(args, kwargs) File "/home/work/anaconda3/envs/pytorch/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 364, in embedding 'padding_idx': padding_idx File "/home/work/paddle/transformer/model.py", line 291, in prepare_encoder_decoder initializer=fluid.initializer.Normal(0., src_emb_dim-0.5))) File "/home/work/paddle/transformer/model.py", line 652, in wrap_encoder word_emb_param_name=word_emb_param_names[0]) File "/home/work/paddle/transformer/model.py", line 581, in transformer enc_inputs, ) File "train.py", line 665, in train is_test=False) File "train.py", line 772, in train(args) C++ Callstacks: Enforce failed. Expected ids[i] < row_number, but received ids[i]:36548 >= row_number:36532. at [/paddle/paddle/fluid/operators/lookup_table_op.h:86] PaddlePaddle Call Stacks: 0 0x7fcece6f1e0dp void paddle::platform::EnforceNotMet::Initstd::string(std::string, char const, int) + 365 1 0x7fcece6f2157p paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) + 87 2 0x7fceceb180fap paddle::operators::LookupTableKernel::Compute(paddle::framework::ExecutionContext const&) const + 2970 3 0x7fceceb18533p std::Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CPUPlace, false, 0ul, paddle::operators::LookupTableKernel, paddle::operators::LookupTableKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1 (closed)}>::M_invoke(std::Any_data const&, paddle::framework::ExecutionContext const&) + 35 4 0x7fced00ffd03p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 659 5 0x7fced00fd575p paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 341 6 0x7fcecff73bd9p 7 0x7fcecff6c081p paddle::framework::details::OpHandleBase::RunAndRecordEvent(std::function<void ()> const&) + 769 8 0x7fcecff7386cp paddle::framework::details::ComputationOpHandle::RunImpl() + 124 9 0x7fcecff6cfc6p paddle::framework::details::OpHandleBase::Run(bool) + 118 10 0x7fcecff570bap 11 0x7fcecf2c53d3p std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) + 35 12 0x7fcecf2886f7p std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&) + 39 13 0x7fcf4788bbe0p pthread_once + 80 14 0x7fcecff55b02p 15 0x7fcecf289b24p ThreadPool::ThreadPool(unsigned long)::{lambda()#1 (closed)}::operator()() const + 404 16 0x7fcef41f1678p 17 0x7fcf47886df3p 18 0x7fcf475b42cdp clone + 109

terminate called without an active exception *** Aborted at 1552404396 (unix time) try "date -d @1552404396" if you are using GNU date *** PC: @ 0x0 (unknown) *** SIGABRT (@0x1f400005314) received by PID 21268 (TID 0x7fc9371de700) from PID 21268; stack trace: *** @ 0x7fcf4788e130 (unknown) @ 0x7fcf474f39d9 __GI_raise @ 0x7fcf474f50e8 __GI_abort @ 0x7fcef41d73df __gnu_cxx::__verbose_terminate_handler() @ 0x7fcef41d5b16 __cxxabiv1::__terminate() @ 0x7fcef41d5b4c std::terminate() @ 0x7fcef41d581e __gxx_personality_v0 @ 0x7fcf3e975045 _Unwind_ForcedUnwind_Phase2 @ 0x7fcf3e97531b _Unwind_ForcedUnwind @ 0x7fcf4788cd90 __GI___pthread_unwind @ 0x7fcf47887e15 __pthread_exit @ 0x7fcf47e876f9 PyThread_exit_thread @ 0x7fcf47d20d82 PyEval_RestoreThread.cold.739 @ 0x7fcece70a4d9 pybind11::gil_scoped_release::~gil_scoped_release() @ 0x7fcece6d3c43 ZZN8pybind1112cpp_function10initializeIZN6paddle6pybindL18pybind11_init_coreERNS_6moduleEEUlRNS2_9operators6reader22LoDTensorBlockingQueueERKSt6vectorINS2_9framework9LoDTensorESaISC_EEE60_bIS9_SG_EINS_4nameENS_9is_methodENS_7siblingEEEEvOT_PFT0_DpT1_EDpRKT2_ENUlRNS_6detail13function_callEE1_4_FUNESY @ 0x7fcece71cc9e pybind11::cpp_function::dispatcher() @ 0x7fcf47dcf744 _PyCFunction_FastCallDict @ 0x7fcf47e5657e call_function @ 0x7fcf47e7b38a _PyEval_EvalFrameDefault @ 0x7fcf47e518c6 PyEval_EvalCodeEx @ 0x7fcf47e521a6 function_call @ 0x7fcf47dcf54e PyObject_Call @ 0x7fcf47e7ca6c _PyEval_EvalFrameDefault @ 0x7fcf47e5053b fast_function @ 0x7fcf47e56505 call_function @ 0x7fcf47e7b38a _PyEval_EvalFrameDefault @ 0x7fcf47e5053b fast_function @ 0x7fcf47e56505 call_function @ 0x7fcf47e7b38a _PyEval_EvalFrameDefault @ 0x7fcf47e50bab _PyFunction_FastCallDict @ 0x7fcf47dcfb0f _PyObject_FastCallDict @ 0x7fcf47dd46a3 _PyObject_Call_Prepend