Created by: qingqing01

======================================================================
[14:07:21]	[Step 1/1] ERROR: test_simple_fc (test_parallel_executor.TestMNIST)
[14:07:21]	[Step 1/1] ----------------------------------------------------------------------
[14:07:21]	[Step 1/1] Traceback (most recent call last):
[14:07:21]	[Step 1/1]   File "test_parallel_executor.py", line 268, in test_simple_fc
[14:07:21]	[Step 1/1]     "label": label})
[14:07:21]	[Step 1/1]   File "test_parallel_executor.py", line 228, in check_network_convergence
[14:07:21]	[Step 1/1]     exe.run([], feed_dict=feed_dict)
[14:07:21]	[Step 1/1]   File "/paddle/build/python/paddle/fluid/parallel_executor.py", line 145, in run
[14:07:21]	[Step 1/1]     self.executor.run(fetch_list, fetch_var_name, feed_tensor_dict)
[14:07:21]	[Step 1/1] EnforceNotMet: cudaMemcpyPeerAsync failed in paddle::platform::GpuMemcpyPeer: an illegal memory access was encountered at [/paddle/paddle/fluid/platform/gpu_info.cc:134]
[14:07:21]	[Step 1/1] PaddlePaddle Call Stacks: 
[14:07:21]	[Step 1/1] 0       0x7fe3289425dcp paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 572
[14:07:21]	[Step 1/1] 1       0x7fe329842028p paddle::platform::GpuMemcpyPeer(void*, int, void const*, int, unsigned long, CUstream_st*) + 296
[14:07:21]	[Step 1/1] 2       0x7fe32975e9d3p paddle::framework::TensorCopy(paddle::framework::Tensor const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::platform::DeviceContext const&, paddle::framework::Tensor*) + 3107
[14:07:21]	[Step 1/1] 3       0x7fe32975f318p paddle::framework::TensorCopy(paddle::framework::Tensor const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::framework::Tensor*) + 104
[14:07:21]	[Step 1/1] 4       0x7fe3296ec6f2p paddle::framework::LoDTensor::SplitLoDTensor(std::vector<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::allocator<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > >) const + 2434
[14:07:21]	[Step 1/1] 5       0x7fe328a170ddp paddle::framework::ParallelExecutor::SplitTensorToPlaces(std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, paddle::framework::LoDTensor, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, paddle::framework::LoDTensor> > > const&) + 413
[14:07:21]	[Step 1/1] 6       0x7fe328a17e35p paddle::framework::ParallelExecutor::Run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, paddle::framework::LoDTensor, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, paddle::framework::LoDTensor> > > const&) + 69