Created by: daming-lu

This could be due to not enough memory in GPU

Traceback (most recent call last):
  File "train.py", line 33, in <module>
    word_dict = paddle.dataset.imikolov.build_dict()
  File "train.py", line 185, in main
    params_dirname=params_dirname)
  File "train.py", line 130, in train
    feed_order=['firstw', 'secondw', 'thirdw', 'fourthw', 'nextw'])
  File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/trainer.py", line 246, in train
    feed_order)
  File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/trainer.py", line 294, in _train_by_executor
    self._train_by_any_executor(event_handler, exe, num_epochs, reader)
  File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/trainer.py", line 308, in _train_by_any_executor
    for var in self.train_func_outputs
  File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/executor.py", line 340, in run
    self.executor.run(program.desc, scope, 0, True, True)
paddle.fluid.core.EnforceNotMet: an illegal memory access was encountered at [/paddle/paddle/fluid/platform/device_context.cc:179]
PaddlePaddle Call Stacks: 
0       0x7f0ad536c286p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 486
1       0x7f0ad61c5f55p paddle::platform::CUDADeviceContext::Wait() const + 549
2       0x7f0ad5f0427dp paddle::framework::Vector<short>::WaitPlace(boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>) + 141
3       0x7f0ad5f04988p paddle::framework::Vector<short>::ImmutableCUDA(boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>) const + 712
4       0x7f0ad5f04b88p paddle::framework::Vector<short>::CUDAMutableData(boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>) + 136
5       0x7f0ad5f07b78p paddle::operators::math::ConcatGradFunctor<paddle::platform::CUDADeviceContext, float>::operator()(paddle::platform::CUDADeviceContext const&, paddle::framework::Tensor const&, int, std::vector<paddle::framework::Tensor, std::allocator<paddle::framework::Tensor> >*) + 728
6       0x7f0ad5d2eea8p paddle::operators::ConcatGradKernel<paddle::platform::CUDADeviceContext, float>::Compute(paddle::framework::ExecutionContext const&) const + 968
7       0x7f0ad6061506p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 1318
8       0x7f0ad54070c7p paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool) + 263
9       0x7f0ad5408164p paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool) + 100
10      0x7f0ad538538bp void pybind11::cpp_function::initialize<pybind11::cpp_function::initialize<void, paddle::framework::Executor, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(void (paddle::framework::Executor::*)(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool)#1}, void, paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(pybind11::cpp_function::initialize<void, paddle::framework::Executor, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(void (paddle::framework::Executor::*)(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool)#1}&&, void (*)(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call) + 555
11      0x7f0ad537da44p pybind11::cpp_function::dispatcher(_object*, _object*, _object*) + 2596
12            0x4c37edp PyEval_EvalFrameEx + 31165
13            0x4b9ab6p PyEval_EvalCodeEx + 774
14            0x4c16e7p PyEval_EvalFrameEx + 22711
15            0x4c136fp PyEval_EvalFrameEx + 21823
16            0x4c136fp PyEval_EvalFrameEx + 21823
17            0x4b9ab6p PyEval_EvalCodeEx + 774
18            0x4c16e7p PyEval_EvalFrameEx + 22711
19            0x4b9ab6p PyEval_EvalCodeEx + 774
20            0x4c1e6fp PyEval_EvalFrameEx + 24639
21            0x4b9ab6p PyEval_EvalCodeEx + 774
22            0x4c1e6fp PyEval_EvalFrameEx + 24639
23            0x4b9ab6p PyEval_EvalCodeEx + 774
24            0x4eb30fp
25            0x4e5422p PyRun_FileExFlags + 130
26            0x4e3cd6p PyRun_SimpleFileExFlags + 390
27            0x493ae2p Py_Main + 1554
28      0x7f0b7a74d830p __libc_start_main + 240
29            0x4933e9p _start + 41