Transformer Fails when MKLDNN is used
Created by: kbinias
This commit made Transformer (used with MKL-DNN) crashed: a872eb90
@panyx0718 Could you please take a look and advice on this issue?
if there is a problem with reproduction of issue or more information is needed, feel free to ask.
Example log & reference command line for reproduction:
FLAGS_use_mkldnn=True OMP_NUM_THREADS=1 python -u infer.py --src_vocab_fpath /nfs/fm/home/jczaja/data/wmt16_ende_data_bpe_clean/vocab_all.bpe.32000 --trg_vocab_fpath /nfs/fm/home/jczaja/data/wmt16_ende_data_bpe_clean/vocab_all.bpe.32000 --special_token '<s>' '<e>' '<unk>' --test_file_pattern /nfs/fm/home/jczaja/data/wmt16_ende_data_bpe_clean/newstest2016.tok.bpe.32000.en-de --token_delimiter ' ' model_path /nfs/fm/home/jczaja/models/iter_100000.infer.model beam_size 4 max_out_len 255
----------- Configuration Arguments -----------
batch_size: 1
device: CPU
display_output: False
num_profiling_passes: 100
opts: ['model_path', '/nfs/fm/home/jczaja/models/iter_100000.infer.model', 'beam_size', '4', 'max_out_len', '255']
pool_size: 10000
profile: True
skip_pass_num: 0
special_token: ['<s>', '<e>', '<unk>']
src_vocab_fpath: /nfs/fm/home/jczaja/data/wmt16_ende_data_bpe_clean/vocab_all.bpe.32000
test_file_pattern: /nfs/fm/home/jczaja/data/wmt16_ende_data_bpe_clean/newstest2016.tok.bpe.32000.en-de
token_delimiter:
trg_vocab_fpath: /nfs/fm/home/jczaja/data/wmt16_ende_data_bpe_clean/vocab_all.bpe.32000
------------------------------------------------
Traceback (most recent call last):
File "infer-profile.py", line 308, in <module>
infer(args)
File "infer-profile.py", line 296, in infer
inferencer(test_data, trg_idx2word)
File "infer-profile.py", line 208, in fast_infer
return_numpy=False)
File "/home/jczaja/paddle/virtualpython/local/lib/python2.7/site-packages/paddle/fluid/executor.py", line 472, in run
self.executor.run(program.desc, scope, 0, True, True)
paddle.fluid.core.EnforceNotMet: Enforce failed. Expected x_rank == axis_size, but received x_rank:3 != axis_size:4.
The input tensor's rank(3) should be equal to the axis's size(4) at [/home/jczaja/paddle/paddle/fluid/operators/transpose_op.cc:43]
PaddlePaddle Call Stacks:
0 0x7fd2a422ec7dp void paddle::platform::EnforceNotMet::Init<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, char const*, int) + 509
1 0x7fd2a4230042p paddle::platform::EnforceNotMet::EnforceNotMet<char const*, char const*, char const*, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >(char const*, int, char const*, char const*, char const*, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 162
2 0x7fd2a49b0b95p paddle::operators::TransposeOp::InferShape(paddle::framework::InferShapeContext*) const + 1397
3 0x7fd2a49b12b2p paddle::operators::Transpose2Op::InferShape(paddle::framework::InferShapeContext*) const + 50
4 0x7fd2a500a3c7p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 599
5 0x7fd2a5005127p paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 535
6 0x7fd2a431af77p paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 295
7 0x7fd2a4de84b5p paddle::operators::WhileOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 981
8 0x7fd2a5005127p paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 535
9 0x7fd2a431af77p paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 295
10 0x7fd2a431d9fap paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool) + 218
11 0x7fd2a42060fap
12 0x7fd2a425225dp
13 0x4cada2p PyEval_EvalFrameEx + 28066
14 0x4c2765p PyEval_EvalCodeEx + 597
15 0x4ca099p PyEval_EvalFrameEx + 24729
16 0x4c9d8fp PyEval_EvalFrameEx + 23951
17 0x4c2765p PyEval_EvalCodeEx + 597
18 0x4ca099p PyEval_EvalFrameEx + 24729
19 0x4c2765p PyEval_EvalCodeEx + 597
20 0x4c2509p PyEval_EvalCode + 25
21 0x4f1defp
22 0x4ec652p PyRun_FileExFlags + 130
23 0x4eae31p PyRun_SimpleFileExFlags + 401
24 0x49e14ap Py_Main + 1674
25 0x7fd2c0644830p __libc_start_main + 240
26 0x49d9d9p _start + 41