Created by: Melonzhou

1.6.1 CPU版本 CPU_NUM = 1 和4时正常训练，不会出错。当CPU_NUM设置成2时，MergeLoDTensor出错 报错信息 INFO: 12-20 13:55:16 trainer.py [train_log:283][5073] train log: steps 5, loss 56.21187973022461, metrics: {'acc': 0.0} INFO: 12-20 13:55:24 trainer.py [train_log:283][5073] train log: steps 10, loss 18.280614852905273, metrics: {'acc': 0.0} /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/paddle/fluid/executor.py:774: UserWarning: The following exception is not an EOF exception. "The following exception is not an EOF exception.") Traceback (most recent call last): File "/Users/baidu/workspace/zhouyuan/text_one/baidu/nlp/textone/tasks/text2sql/script/text2sql_train.py", line 49, in trainer.train_and_eval() File "/Users/baidu/workspace/zhouyuan/text_one/baidu/nlp/textone/textone/training/trainer.py", line 237, in train_and_eval metrics_tensor_value = self.train_exe.run(fetch_list=self.train_metrics_tensor_name) File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/paddle/fluid/parallel_executor.py", line 311, in run return_numpy=return_numpy) File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/paddle/fluid/executor.py", line 775, in run six.reraise(sys.exc_info()) File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/six.py", line 693, in reraise raise value File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/paddle/fluid/executor.py", line 770, in run use_program_cache=use_program_cache) File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/paddle/fluid/executor.py", line 829, in _run_impl return_numpy=return_numpy) File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/paddle/fluid/executor.py", line 669, in _run_parallel tensors = exe.run(fetch_var_names)._move_to_list() paddle.fluid.core_avx.EnforceNotMet: 0 std::__1::basic_string<char, std::__1::char_traits, std::__1::allocator > paddle::platform::GetTraceBackString<std::__1::basic_string<char, std::__1::char_traits, std::__1::allocator > const&>(std::__1::basic_string<char, std::__1::char_traits, std::__1::allocator > const&&&, char const, int) 1 paddle::framework::LoDTensor::MergeLoDTensor(std::1::vector<paddle::framework::LoDTensor const*, std::1::allocator<paddle::framework::LoDTensor const*> > const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>) 2 paddle::framework::details::FetchOpHandle::WaitAndMergeCPUTensors() const 3 paddle::framework::details::FetchOpHandle::RunImpl() 4 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 5 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::__1::shared_ptr<paddle::framework::BlockingQueue > const&, unsigned long*) 6 std::__1::__packaged_task_func<std::__1::__bind<paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpAsync(std::__1::unordered_map<paddle::framework::details::OpHandleBase*, std::__1::atomic, std::__1::hashpaddle::framework::details::OpHandleBase*, std::__1::equal_topaddle::framework::details::OpHandleBase*, std::__1::allocator<std::__1::pair<paddle::framework::details::OpHandleBase* const, std::__1::atomic > > >, paddle::framework::details::OpHandleBase, std::__1::shared_ptr<paddle::framework::BlockingQueue > const&)::$_0>, std::__1::allocator<std::__1::__bind<paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpAsync(std::__1::unordered_map<paddle::framework::details::OpHandleBase*, std::__1::atomic, std::__1::hashpaddle::framework::details::OpHandleBase*, std::__1::equal_topaddle::framework::details::OpHandleBase*, std::__1::allocator<std::__1::pair<paddle::framework::details::OpHandleBase* const, std::__1::atomic > > >, paddle::framework::details::OpHandleBase, std::__1::shared_ptr<paddle::framework::BlockingQueue > const&)::$_0> >, void ()>::operator()() 7 std::__1::packaged_task<void ()>::operator()() 8 ThreadPool::ThreadPool(unsigned long)::'lambda'()::operator()() const 9 std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_deletestd::__1::__thread_struct >, ThreadPool::ThreadPool(unsigned long)::'lambda'()> >(void*, void*) PaddleCheckError: Expected framework::product(new_dim) / new_dim[0] == framework::product(t->dims()) / t->dims()[0], but received framework::product(new_dim) / new_dim[0]:12 != framework::product(t->dims()) / t->dims()[0]:18. at [/home/teamcity/work/ef54dc8a5b211854/paddle/fluid/framework/lod_tensor.cc:363]

Process finished with exit code 255

代码片段： metrics_tensor_value = self.train_exe.run(fetch_list=self.train_metrics_tensor_name) self.train_exe是ParallelExecutor self.train_metrics_tensor_name = [decode_output, label, loss]