Created by: yuzunrui

同样的代码，epoch=3时正常训练完并保存模型，epoch=5时训练完保存模型阶段报错（报错能复现），报错信息如下：

09/17/2019 10:42:06 - INFO - __main__ - epoch: 4, progress: 78746/78746, step: 49190, loss: 1.334699, speed: 0.588579 steps/s
09/17/2019 10:42:23 - INFO - __main__ - epoch: 4, progress: 78746/78746, step: 49200, loss: 0.997795, speed: 0.589185 steps/s
09/17/2019 10:42:40 - INFO - __main__ - epoch: 4, progress: 78746/78746, step: 49210, loss: 2.481894, speed: 0.588316 steps/s

An exception was thrown!
Invoke operator scale error.
Python Call stacks:
File "/home/slurm/job/tmp/job-130809/python/lib/python3.7/site-packages/paddle/fluid/framework.py", line 1780, in append_op
attrs=kwargs.get("attrs", None))
File "/home/slurm/job/tmp/job-130809/python/lib/python3.7/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op
return self.main_program.current_block().append_op(*args, **kwargs)
File "/home/slurm/job/tmp/job-130809/python/lib/python3.7/site-packages/paddle/fluid/layers/nn.py", line 10410, in scale
'bias_after_scale': bias_after_scale
File "/home/slurm/job/tmp/job-130809/src/model/bert.py", line 134, in _build_model
self_attn_mask = fluid.layers.scale(x=input_mask, scale=10000.0, bias=-1.0, bias_after_scale=False)
File "/home/slurm/job/tmp/job-130809/src/model/bert.py", line 89, in __init__
self._build_model(src_ids, position_ids, sentence_ids, input_mask)
File "src/run_squad.py", line 133, in create_model
use_masklm_feedforward=args.use_masklm_feedforward)
File "src/run_squad.py", line 277, in train
is_training=True)
File "src/run_squad.py", line 505, in <module>
train(args)
C++ Call stacks:
holder_ should not be null
Tensor not initialized yet when Tensor::place() is called. at [/paddle/paddle/fluid/framework/tensor.h:133]
PaddlePaddle Call Stacks:
0 0x7fa5bceeafb8p void paddle::platform::EnforceNotMet::Init<std::string>(std::string, char const*, int) + 360
1 0x7fa5bceeb307p paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) + 87
2 0x7fa5bceec4c3p paddle::framework::Tensor::place() const + 131
3 0x7fa5bd98385bp paddle::operators::ScaleKernel<paddle::platform::CUDADeviceContext, float>::Compute(paddle::framework::ExecutionContext const&) const + 411
4 0x7fa5bd9840e3p std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::ScaleKernel<paddle::platform::CUDADeviceContext, float>, paddle::operators::ScaleKernel<paddle::platform::CUDADeviceContext, double>, paddle::operators::ScaleKernel<paddle::platform::CUDADeviceContext, int>, paddle::operators::ScaleKernel<paddle::platform::CUDADeviceContext, long>, paddle::operators::ScaleKernel<paddle::platform::CUDADeviceContext, paddle::platform::float16> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) + 35
5 0x7fa5befff93bp paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::framework::RuntimeContext*) const + 411
6 0x7fa5beffff31p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 529
7 0x7fa5beffa7b3p paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 307
8 0x7fa5bedfd226p paddle::framework::details::ComputationOpHandle::RunImpl() + 166
9 0x7fa5bed37c2cp paddle::framework::details::ThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) + 316
10 0x7fa5bed32a25p
11 0x7fa5bd12c6a3p std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) + 35
12 0x7fa5bcfb4527p std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&) + 39
13 0x7fa5f4ac4b23p pthread_once + 83
14 0x7fa5bed32512p
15 0x7fa5bcfb5aa4p ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const + 404
16 0x7fa5deed4678p
17 0x7fa5f4abf851p
18 0x7fa5f480d67dp clone + 109

Traceback (most recent call last):
File "src/run_squad.py", line 505, in <module>
train(args)
File "src/run_squad.py", line 396, in train
outputs = train_exe.run(fetch_list=fetch_list)
File "/home/slurm/job/tmp/job-130809/python/lib/python3.7/site-packages/paddle/fluid/parallel_executor.py", line 279, in run
return_numpy=return_numpy)
File "/home/slurm/job/tmp/job-130809/python/lib/python3.7/site-packages/paddle/fluid/executor.py", line 615, in run
six.reraise(*sys.exc_info())
File "/home/slurm/job/tmp/job-130809/python/lib/python3.7/site-packages/six.py", line 693, in reraise
raise value
File "/home/slurm/job/tmp/job-130809/python/lib/python3.7/site-packages/paddle/fluid/executor.py", line 611, in run
use_program_cache=use_program_cache)
File "/home/slurm/job/tmp/job-130809/python/lib/python3.7/site-packages/paddle/fluid/executor.py", line 663, in _run_impl
return_numpy=return_numpy)
File "/home/slurm/job/tmp/job-130809/python/lib/python3.7/site-packages/paddle/fluid/executor.py", line 530, in _run_parallel
tensors = exe.run(fetch_var_names)._move_to_list()
paddle.fluid.core_avx.EnforceNotMet: Invoke operator scale error.
Python Call stacks:
File "/home/slurm/job/tmp/job-130809/python/lib/python3.7/site-packages/paddle/fluid/framework.py", line 1780, in append_op
attrs=kwargs.get("attrs", None))
File "/home/slurm/job/tmp/job-130809/python/lib/python3.7/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op
return self.main_program.current_block().append_op(*args, **kwargs)
File "/home/slurm/job/tmp/job-130809/python/lib/python3.7/site-packages/paddle/fluid/layers/nn.py", line 10410, in scale
'bias_after_scale': bias_after_scale
File "/home/slurm/job/tmp/job-130809/src/model/bert.py", line 134, in _build_model
self_attn_mask = fluid.layers.scale(x=input_mask, scale=10000.0, bias=-1.0, bias_after_scale=False)
File "/home/slurm/job/tmp/job-130809/src/model/bert.py", line 89, in __init__
self._build_model(src_ids, position_ids, sentence_ids, input_mask)
File "src/run_squad.py", line 133, in create_model
use_masklm_feedforward=args.use_masklm_feedforward)
File "src/run_squad.py", line 277, in train
is_training=True)
File "src/run_squad.py", line 505, in <module>
train(args)
C++ Call stacks:
holder_ should not be null
Tensor not initialized yet when Tensor::place() is called. at [/paddle/paddle/fluid/framework/tensor.h:133]
PaddlePaddle Call Stacks:
0 0x7fa5bceeafb8p void paddle::platform::EnforceNotMet::Init<std::string>(std::string, char const*, int) + 360
1 0x7fa5bceeb307p paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) + 87
2 0x7fa5bceec4c3p paddle::framework::Tensor::place() const + 131
3 0x7fa5bd98385bp paddle::operators::ScaleKernel<paddle::platform::CUDADeviceContext, float>::Compute(paddle::framework::ExecutionContext const&) const + 411
4 0x7fa5bd9840e3p std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::ScaleKernel<paddle::platform::CUDADeviceContext, float>, paddle::operators::ScaleKernel<paddle::platform::CUDADeviceContext, double>, paddle::operators::ScaleKernel<paddle::platform::CUDADeviceContext, int>, paddle::operators::ScaleKernel<paddle::platform::CUDADeviceContext, long>, paddle::operators::ScaleKernel<paddle::platform::CUDADeviceContext, paddle::platform::float16> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) + 35
5 0x7fa5befff93bp paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::framework::RuntimeContext*) const + 411
6 0x7fa5beffff31p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 529
7 0x7fa5beffa7b3p paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 307
8 0x7fa5bedfd226p paddle::framework::details::ComputationOpHandle::RunImpl() + 166
9 0x7fa5bed37c2cp paddle::framework::details::ThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) + 316
10 0x7fa5bed32a25p
11 0x7fa5bd12c6a3p std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) + 35
12 0x7fa5bcfb4527p std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&) + 39
13 0x7fa5f4ac4b23p pthread_once + 83
14 0x7fa5bed32512p
15 0x7fa5bcfb5aa4p ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const + 404
16 0x7fa5deed4678p
17 0x7fa5f4abf851p
18 0x7fa5f480d67dp clone + 109