Created by: zishuaiz

paddle版本0.14，机器版本：cent4 在cpu版本下，Trainer配置了parallel=True，可以启动，也能看到cpu变化，但是没过2分钟就出core了

~/opt/py27pdcpu/bin/python train.py
/home/xxxx/opt/py27pdcpu/lib/python2.7/site-packages/sklearn/externals/joblib/_multiprocessing_helpers.py:28: UserWarning: This platform lacks a functioning sem_open implementation, therefore, the required synchronization primitives needed will not function, see issue 3770..  joblib will operate in serial mode
  warnings.warn('%s.  joblib will operate in serial mode' % (e,))
Traceback (most recent call last):
  File "train.py", line 132, in <module>
    main(use_cuda)
  File "train.py", line 127, in main
    train(use_cuda)
  File "train.py", line 123, in train
    feed_order=feed_order)
  File "/home/xxx/opt/py27pdcpu/lib/python2.7/site-packages/paddle/fluid/trainer.py", line 410, in train
    feed_order)
  File "/home/xxx/opt/py27pdcpu/lib/python2.7/site-packages/paddle/fluid/trainer.py", line 535, in _train_by_parallel_executor
    self._train_by_any_executor(event_handler, pe, num_epochs, reader)
  File "/home/xxx/opt/py27pdcpu/lib/python2.7/site-packages/paddle/fluid/trainer.py", line 498, in _train_by_any_executor
    for var in self.train_func_outputs
  File "/home/xxx/opt/py27pdcpu/lib/python2.7/site-packages/paddle/fluid/parallel_executor.py", line 269, in run
    self.executor.run(fetch_list, fetch_var_name)
paddle.fluid.core.EnforceNotMet: enforce posix_memalign(&p, 4096ul, size) == 0 failed, 12 != 0
Alloc 4225602176 error! at [/paddle/paddle/fluid/memory/detail/system_allocator.cc:52]
PaddlePaddle Call Stacks:
0       0x7f047fa8c376p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 486
1       0x7f04803d855ap paddle::memory::detail::CPUAllocator::Alloc(unsigned long*, unsigned long) + 378
2       0x7f04803d6075p paddle::memory::detail::BuddyAllocator::RefillPool() + 53
3       0x7f04803d688cp paddle::memory::detail::BuddyAllocator::Alloc(unsigned long) + 716
4       0x7f04803116b5p void* paddle::memory::Alloc<paddle::platform::CPUPlace>(paddle::platform::CPUPlace, unsigned long) + 181
5       0x7f048030b971p paddle::framework::Tensor::mutable_data(boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::type_index) + 433
6       0x7f048030d677p paddle::framework::TensorCopy(paddle::framework::Tensor const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::platform::DeviceContext const&, paddle::framework::Tensor*) + 455
7       0x7f047fbb47c8p paddle::operators::WriteToArrayOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 1176
8       0x7f04802a5450p paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 208
9       0x7f047fb1fcdfp paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 255
10      0x7f047fb20d30p paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool) + 128
11      0x7f047fba9ae8p paddle::operators::WhileOp::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 824
12      0x7f04802a5450p paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 208
13      0x7f0480219c5cp
14      0x7f048021989cp paddle::framework::details::ComputationOpHandle::RunImpl() + 92
15      0x7f048023b98cp paddle::framework::details::OpHandleBase::Run(bool) + 28
16      0x7f0480233a9ap
17      0x7f0480124993p std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) + 35
18      0x7f0480125127p std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&) + 39
19      0x7f0501aea973p pthread_once + 83
20      0x7f04802326e2p
21      0x7f0480236f36p std::thread::_Impl<std::_Bind_simple<ThreadPool::ThreadPool(unsigned long)::{lambda()#1} ()> >::_M_run() + 406
22      0x7f048e0dd8a0p
23      0x7f0501ae51c3p
24      0x7f050110d12dp clone + 109