test_machine_translation.py gets a crash dump in GPU mode
Created by: lcy-seso
I run this example: https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/fluid/tests/book/test_machine_translation.py
I only change this line: place = core.CPUPlace()
into:
place = core.GPUPlace(0)
The error information is as follows:
terminate called after throwing an instance of 'thrust::system::system_error'
what(): an illegal memory access was encountered
Aborted (core dumped)
Backtrace in gdb :
#0 0x000000318ae328a5 in raise () from /lib64/libc.so.6
#1 0x000000318ae34085 in abort () from /lib64/libc.so.6
#2 0x00007faad387d705 in __gnu_cxx::__verbose_terminate_handler() () from /home/caoying/.jumbo/opt/gcc48/lib64/libstdc++.so.6
#3 0x00007faad387b8a6 in __cxxabiv1::__terminate(void (*)()) () from /home/caoying/.jumbo/opt/gcc48/lib64/libstdc++.so.6
#4 0x00007faad387a969 in __cxa_call_terminate () from /home/caoying/.jumbo/opt/gcc48/lib64/libstdc++.so.6
#5 0x00007faad387b51a in __gxx_personality_v0 () from /home/caoying/.jumbo/opt/gcc48/lib64/libstdc++.so.6
#6 0x00007faaec4df813 in _Unwind_RaiseException_Phase2 (exc=exc@entry=0x122b4f20, context=context@entry=0x7fff1bb73680) at /home/caoying/.jumbo/var/jumbo/tmp/jumbo.KVlu7OwtI/gcc48/src/gcc-4.8.3/libgcc/unwind.inc:62
#7 0x00007faaec4dfbab in _Unwind_RaiseException (exc=0x122b4f20) at /home/caoying/.jumbo/var/jumbo/tmp/jumbo.KVlu7OwtI/gcc48/src/gcc-4.8.3/libgcc/unwind.inc:131
#8 0x00007faad387bae5 in __cxa_throw () from /home/caoying/.jumbo/opt/gcc48/lib64/libstdc++.so.6
#9 0x00007faabee98b95 in thrust::system::cuda::experimental::pinned_allocator<unsigned long>::deallocate (this=this@entry=0x4d, cnt=<optimized out>, p=<optimized out>)
at /home/work/cuda-7.5/include/thrust/system/cuda/experimental/pinned_allocator.h:182
#10 thrust::detail::allocator_traits<thrust::system::cuda::experimental::pinned_allocator<unsigned long> >::deallocate(thrust::system::cuda::experimental::pinned_allocator<unsigned long>&, unsigned long*, unsigned long)::workaround_warnings::deallocate(thrust::system::cuda::experimental::pinned_allocator<unsigned long>&, unsigned long*, unsigned long) (a=..., n=<optimized out>, p=<optimized out>)
at /home/work/cuda-7.5/include/thrust/detail/allocator/allocator_traits.inl:256
#11 thrust::detail::allocator_traits<thrust::system::cuda::experimental::pinned_allocator<unsigned long> >::deallocate (a=..., n=<optimized out>, p=<optimized out>)
at /home/work/cuda-7.5/include/thrust/detail/allocator/allocator_traits.inl:260
#12 thrust::detail::contiguous_storage<unsigned long, thrust::system::cuda::experimental::pinned_allocator<unsigned long> >::deallocate (this=this@entry=0x7fff1bb73a00)
at /home/work/cuda-7.5/include/thrust/detail/contiguous_storage.inl:172
#13 0x00007faabee9af4c in thrust::detail::contiguous_storage<unsigned long, thrust::system::cuda::experimental::pinned_allocator<unsigned long> >::deallocate (this=0x7fff1bb73a00)
at /home/caoying/.jumbo/opt/gcc48/include/c++/4.8.3/ext/new_allocator.h:110
#14 thrust::detail::contiguous_storage<unsigned long, thrust::system::cuda::experimental::pinned_allocator<unsigned long> >::~contiguous_storage (this=0x7fff1bb73a00, __in_chrg=<optimized out>)
at /home/work/cuda-7.5/include/thrust/detail/contiguous_storage.inl:64
#15 thrust::detail::vector_base<unsigned long, thrust::system::cuda::experimental::pinned_allocator<unsigned long> >::~vector_base (this=0x7fff1bb73a00, __in_chrg=<optimized out>)
at /home/work/cuda-7.5/include/thrust/detail/vector_base.inl:475
#16 thrust::host_vector<unsigned long, thrust::system::cuda::experimental::pinned_allocator<unsigned long> >::~host_vector (this=0x7fff1bb73a00, __in_chrg=<optimized out>) at /home/work/cuda-7.5/include/thrust/host_vector.h:52
#17 paddle::framework::AppendLoD (lod=lod@entry=0x122ba268, lod_length=...) at /home/caoying/paddle_codes/paddle_github/paddle/framework/lod_tensor.cc:180
#18 0x00007faabee02c1b in paddle::operators::LoDTensorToArrayOp::Run (this=<optimized out>, scope=..., dev_ctx=...) at /home/caoying/paddle_codes/paddle_github/paddle/operators/lod_tensor_to_array_op.cc:65
#19 0x00007faabe8520ed in paddle::framework::Executor::Run (this=<optimized out>, pdesc=..., scope=0x395bee0, block_id=<optimized out>, create_local_scope=<optimized out>, create_vars=<optimized out>)
at /home/caoying/paddle_codes/paddle_github/paddle/framework/executor.cc:110
#20 0x00007faabe7cfa6c in pybind11::cpp_function::cpp_function<void, paddle::framework::Executor, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(void (paddle::framework::Executor::*)(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool)#1}::operator()(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool) const
(args#4=<optimized out>, args#3=<optimized out>, args#2=<optimized out>, args#1=<optimized out>, args#0=..., c=<optimized out>, __closure=<optimized out>)
at /home/caoying/paddle_codes/paddle_github/build/third_party/pybind/src/extern_pybind/include/pybind11/pybind11.h:69
#21 pybind11::detail::argument_loader<paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool>::call_impl<void, pybind11::cpp_function::cpp_function(Return (Class::*)(Arg ...), const Extra& ...) [with Return = void; Class = paddle::framework::Executor; Arg = {const paddle::framework::ProgramDesc&, paddle::framework::Scope*, int, bool, bool}; Extra = {pybind11::name, pybind11::is_method, pybind11::sibling}]::__lambda5&, 0ul, 1ul, 2ul, 3ul, 4ul, 5ul> (f=..., this=0x7fff1bb74120) at /home/caoying/paddle_codes/paddle_github/build/third_party/pybind/src/extern_pybind/include/pybind11/cast.h:1411
#22 pybind11::detail::argument_loader<paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool>::call<void, pybind11::cpp_function::cpp_function(Return (Class::*)(Arg ...), const Extra& ...) [with Return = void; Class = paddle::framework::Executor; Arg = {const paddle::framework::ProgramDesc&, paddle::framework::Scope*, int, bool, bool}; Extra = {pybind11::name, pybind11::is_method, pybind11::sibling}]::__lambda5&> (f=..., this=0x7fff1bb74120) at /home/caoying/paddle_codes/paddle_github/build/third_party/pybind/src/extern_pybind/include/pybind11/cast.h:1393
#23 void pybind11::cpp_function::initialize<pybind11::cpp_function::initialize<void, paddle::framework::Executor, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(void (paddle::framework::Executor::*)(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool)#1}, void, paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(pybind11::cpp_function::initialize<void, paddle::framework::Executor, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(void (paddle::framework::Executor::*)(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool)#1}&&, void (*)(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(pybind11::detail::function_call&)#3}::operator()(pybind11::detail::function_call) const (call=..., __closure=0x0)
at /home/caoying/paddle_codes/paddle_github/build/third_party/pybind/src/extern_pybind/include/pybind11/pybind11.h:148
#24 void pybind11::cpp_function::initialize<pybind11::cpp_function::initialize<void, paddle::framework::Executor, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_metho
I compile fluid both in Debug and Release mode. The problem is the same.