使用 elementwise_mul OP 报 Paddle internal Check failed
Created by: AoZhang
-
环境 1）PaddlePaddle版本：1.6.2 2）CPU：Intel(R) Xeon(R) CPU E5-2620 v2 3）GPU：no 4）系统环境：CentOS 6.3 ，Python版本 3.6.5
-
问题描述 看报错信息，问题出在了
true_values = layers.elementwise_mul(value_enc, condition, axis=0)打印出来 value_enc, condition 的shape 和 dtype 如下：
(-1, 1000, 10) VarType.FP32
(-1, 1) VarType.FP32看样子应该没有问题才对啊。具体出错日志如下：
/home/xx/paddle_env/lib/python3.6/site-packages/paddle/fluid/executor.py:779: UserWarning: The following exception is not an EOF exception.
"The following exception is not an EOF exception.")
Traceback (most recent call last):
File ".../models/grammar/infer_decoder.py", line 592, in <module>
result = exe.run(feed=_data, fetch_list=outputs + (inputs, ))
File "/home/xx/paddle_env/.../.../utils/debug/executor.py", line 47, in run
return self.exe.run(fluid.default_main_program(), *args, **kwargs)
File "/home/xx/paddle_env/lib/python3.6/site-packages/paddle/fluid/executor.py", line 780, in run
six.reraise(*sys.exc_info())
File "/home/xx/paddle_env/lib/python3.6/site-packages/six.py", line 693, in reraise
raise value
File "/home/xx/paddle_env/lib/python3.6/site-packages/paddle/fluid/executor.py", line 775, in run
use_program_cache=use_program_cache)
File "/home/xx/paddle_env/lib/python3.6/site-packages/paddle/fluid/executor.py", line 822, in _run_impl
use_program_cache=use_program_cache)
File "/home/xx/paddle_env/lib/python3.6/site-packages/paddle/fluid/executor.py", line 899, in _run_program
fetch_var_name)
paddle.fluid.core_avx.EnforceNotMet:
--------------------------------------------
C++ Call Stacks (More useful to developers):
--------------------------------------------
0 std::string paddle::platform::GetTraceBackString<std::string const&>(std::string const&, char const*, int)
1 paddle::memory::detail::MetadataCache::load(paddle::memory::detail::MemoryBlock const*) const
2 paddle::memory::detail::MemoryBlock::total_size(paddle::memory::detail::MetadataCache const&) const
3 paddle::memory::detail::MemoryBlock::split(paddle::memory::detail::MetadataCache*, unsigned long)
4 paddle::memory::detail::BuddyAllocator::SplitToAlloc(std::_Rb_tree_const_iterator<std::tuple<unsigned long, unsigned long, void*> >, unsigned long)
5 paddle::memory::detail::BuddyAllocator::Alloc(unsigned long)
6 void* paddle::memory::legacy::Alloc<paddle::platform::CPUPlace>(paddle::platform::CPUPlace const&, unsigned long)
7 paddle::memory::allocation::NaiveBestFitAllocator::AllocateImpl(unsigned long)
8 paddle::memory::allocation::AllocatorFacade::Alloc(paddle::platform::Place const&, unsigned long)
9 paddle::memory::allocation::AllocatorFacade::AllocShared(paddle::platform::Place const&, unsigned long)
10 paddle::memory::AllocShared(paddle::platform::Place const&, unsigned long)
11 paddle::framework::Tensor::mutable_data(paddle::platform::Place, paddle::framework::proto::VarType_Type, unsigned long)
12 paddle::operators::ElementwiseMulKernel<paddle::platform::CPUDeviceContext, float>::Compute(paddle::framework::ExecutionContext const&) const
13 std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CPUPlace, false, 0ul, paddle::operators::ElementwiseMulKernel<paddle::platform::CPUDeviceContext,
float>, paddle::operators::ElementwiseMulKernel<paddle::platform::CPUDeviceContext, double>, paddle::operators::ElementwiseMulKernel<paddle::platform::CPUDeviceContext, int>, paddle::operators::ElementwiseMulKernel<paddle::platform::CPUDe
viceContext, long> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&)
14 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, paddle::platform::Place const&, paddle::framework::RuntimeContext*) const
15 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, paddle::platform::Place const&) const
16 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, paddle::platform::Place const&)
17 paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool)
18 paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, std::vector<std::string, std::allocator<std::string> > const&, bool)
------------------------------------------
Python Call Stacks (More useful to users):
------------------------------------------
File "/home/xx/paddle_env/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2488, in append_op
attrs=kwargs.get("attrs", None))
File "/home/xx/paddle_env/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op
return self.main_program.current_block().append_op(*args, **kwargs)
File "/home/xx/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14055, in _elementwise_op
'use_mkldnn': use_mkldnn})
File "/home/xx/paddle_env/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 14452, in elementwise_mul
return _elementwise_op(LayerHelper('elementwise_mul', **locals()))
File "/home/xx/paddle_env/.../.../models/grammar/nets.py", line 158, in _select_value
true_values = layers.elementwise_mul(value_enc, condition, axis=0)
File "/home/xx/paddle_env/.../.../models/grammar/nets.py", line 245, in grammar_output
select_v_output = _select_value(cond_select_v, inputs, value_enc, value_len, zero_output, ptr_net, grammar)
File ".../models/grammar/infer_decoder.py", line 444, in _output_layer
return self._fn_output(inputs, actions, gmr_mask, self._decode_vocab, self._grammar)
File ".../models/grammar/infer_decoder.py", line 379, in _grammar_step
logits = self._output_layer(logits, actions, gmr_mask)
File ".../models/grammar/infer_decoder.py", line 179, in step
decode_outputs, decode_states = self._grammar_step(cell_outputs, next_cell_states, states, actions, gmr_mask)
File ".../models/grammar/infer_decoder.py", line 576, in <module>
outputs, states, next_input = decoder.step(time=None, inputs=inputs, states=states, actions=actions, gmr_mask=gmr_mask)
----------------------
Error Message Summary:
----------------------
Error: Paddle internal Check failed. (Please help us create a new issue, here we need to find the developer to add a user friendly error message)
[Hint: Expected desc->check_guards() == true, but received desc->check_guards():0 != true:1.] at (/paddle/paddle/fluid/memory/detail/meta_cache.cc:33)
[operator < elementwise_mul > error]
terminate called after throwing an instance of 'paddle::platform::EnforceNotMet'
what():
--------------------------------------------
C++ Call Stacks (More useful to developers):
--------------------------------------------
0 std::string paddle::platform::GetTraceBackString<std::string const&>(std::string const&, char const*, int)
1 paddle::memory::detail::MetadataCache::load(paddle::memory::detail::MemoryBlock const*) const
2 paddle::memory::detail::MemoryBlock::type(paddle::memory::detail::MetadataCache const&) const
3 paddle::memory::detail::BuddyAllocator::Free(void*)
4 void paddle::memory::legacy::Free<paddle::platform::CPUPlace>(paddle::platform::CPUPlace const&, void*, unsigned long)
5 paddle::memory::allocation::NaiveBestFitAllocator::FreeImpl(paddle::memory::allocation::Allocation*)
6 std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release()
7 paddle::framework::Variable::PlaceholderImpl<paddle::framework::LoDTensor>::~PlaceholderImpl()
8 paddle::framework::Scope::~Scope()
9 paddle::framework::Scope::DropKids()
10 paddle::framework::Scope::~Scope()
11 paddle::framework::ScopePool::DeleteScope(paddle::framework::Scope*)
12 paddle::framework::ScopePool::Clear()
----------------------
Error Message Summary:
----------------------
Error: Paddle internal Check failed. (Please help us create a new issue, here we need to find the developer to add a user friendly error message)
[Hint: Expected desc->check_guards() == true, but received desc->check_guards():0 != true:1.] at (/paddle/paddle/fluid/memory/detail/meta_cache.cc:33)
W0117 21:11:13.445998 32227 init.cc:206] *** Aborted at 1579266673 (unix time) try "date -d @1579266673" if you are using GNU date ***
W0117 21:11:13.447896 32227 init.cc:206] PC: @ 0x0 (unknown)
W0117 21:11:13.447973 32227 init.cc:206] *** SIGABRT (@0x1f400007de3) received by PID 32227 (TID 0x7fa3cf61a700) from PID 32227; stack trace: ***
W0117 21:11:13.449512 32227 init.cc:206] @ 0x7fa3cf1f1160 (unknown)
W0117 21:11:13.451095 32227 init.cc:206] @ 0x7fa3ce75f3f7 __GI_raise
W0117 21:11:13.452637 32227 init.cc:206] @ 0x7fa3ce7607d8 __GI_abort
W0117 21:11:13.453466 32227 init.cc:206] @ 0x7fa3862f5c65 __gnu_cxx::__verbose_terminate_handler()
W0117 21:11:13.454200 32227 init.cc:206] @ 0x7fa3862f3e06 __cxxabiv1::__terminate()
W0117 21:11:13.454921 32227 init.cc:206] @ 0x7fa3862f2ec9 __cxa_call_terminate
W0117 21:11:13.455674 32227 init.cc:206] @ 0x7fa3862f3a7a __gxx_personality_v0
W0117 21:11:13.456523 32227 init.cc:206] @ 0x7fa3a15d1853 _Unwind_RaiseException_Phase2
W0117 21:11:13.457386 32227 init.cc:206] @ 0x7fa3a15d1d87 _Unwind_Resume
W0117 21:11:13.458170 32227 init.cc:206] @ 0x7fa376bf100d paddle::memory::detail::BuddyAllocator::Free()
W0117 21:11:13.460266 32227 init.cc:206] @ 0x7fa376be3cb5 paddle::memory::legacy::Free<>()
W0117 21:11:13.461257 32227 init.cc:206] @ 0x7fa376be43d5 paddle::memory::allocation::NaiveBestFitAllocator::FreeImpl()
W0117 21:11:13.461984 32227 init.cc:206] @ 0x7fa3753a4bc9 std::_Sp_counted_base<>::_M_release()
W0117 21:11:13.462761 32227 init.cc:206] @ 0x7fa3753a5658 paddle::framework::Variable::PlaceholderImpl<>::~PlaceholderImpl()
W0117 21:11:13.463474 32227 init.cc:206] @ 0x7fa376bbdadd paddle::framework::Scope::~Scope()
W0117 21:11:13.464776 32227 init.cc:206] @ 0x7fa376bbd9f1 paddle::framework::Scope::DropKids()
W0117 21:11:13.465487 32227 init.cc:206] @ 0x7fa376bbda5d paddle::framework::Scope::~Scope()
W0117 21:11:13.466053 32227 init.cc:206] @ 0x7fa3755c67f6 paddle::framework::ScopePool::DeleteScope()
W0117 21:11:13.466992 32227 init.cc:206] @ 0x7fa3755c6851 paddle::framework::ScopePool::Clear()
W0117 21:11:13.467104 32227 init.cc:206] @ 0x4b123f capsule_dealloc
W0117 21:11:13.467195 32227 init.cc:206] @ 0x49870f free_keys_object
W0117 21:11:13.467284 32227 init.cc:206] @ 0x499aa9 dict_tp_clear
W0117 21:11:13.467351 32227 init.cc:206] @ 0x43d432 collect
W0117 21:11:13.467555 32227 init.cc:206] @ 0x43e0e1 _PyGC_CollectNoFail
W0117 21:11:13.467746 32227 init.cc:206] @ 0x567b54 PyImport_Cleanup
W0117 21:11:13.467803 32227 init.cc:206] @ 0x4228f8 Py_FinalizeEx.part.3
W0117 21:11:13.467970 32227 init.cc:206] @ 0x43c0c5 Py_Main
W0117 21:11:13.468165 32227 init.cc:206] @ 0x41e672 main
W0117 21:11:13.469758 32227 init.cc:206] @ 0x7fa3ce74bbd5 __libc_start_main
W0117 21:11:13.470021 32227 init.cc:206] @ 0x41e731 (unknown)
W0117 21:11:13.471513 32227 init.cc:206] @ 0x0 (unknown)
Command terminated