The index of gather_op should not be empty when the index's rank is 1
Created by: zhengzhe97
使用cascade_rcnn_r200训练的模型，遇到这个错误，有点像 #990 这个issue，但是我没有修改配置文件的proposals部分，min_size在train和test上面都是0.0
Traceback (most recent call last):
File "tools/train.py", line 368, in <module>
main()
File "tools/train.py", line 241, in main
outs = exe.run(compiled_train_prog, fetch_list=train_values)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py", line 1071, in run
six.reraise(*sys.exc_info())
File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/six.py", line 703, in reraise
raise value
File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py", line 1066, in run
return_merged=return_merged)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py", line 1167, in _run_impl
return_merged=return_merged)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/executor.py", line 879, in _run_parallel
tensors = exe.run(fetch_var_names, return_merged)._move_to_list()
paddle.fluid.core_avx.EnforceNotMet:
--------------------------------------------
C++ Call Stacks (More useful to developers):
--------------------------------------------
0 std::string paddle::platform::GetTraceBackString<std::string const&>(std::string const&, char const*, int)
1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int)
2 void paddle::operators::GPUGather<float, int>(paddle::platform::DeviceContext const&, paddle::framework::Tensor const&, paddle::framework::Tensor const&, paddle::framework::Tensor*)
3 paddle::operators::CUDAGenerateProposalsKernel<paddle::platform::CUDADeviceContext, float>::Compute(paddle::framework::ExecutionContext const&) const
4 std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::CUDAGenerateProposalsKernel<paddle::platform::CUDADeviceContext, float> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&)
5 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, paddle::platform::Place const&, paddle::framework::RuntimeContext*) const
6 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, paddle::platform::Place const&) const
7 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, paddle::platform::Place const&)
8 paddle::framework::details::ComputationOpHandle::RunImpl()
9 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*)
10 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr<paddle::framework::BlockingQueue<unsigned long> > const&, unsigned long*)
11 std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&)
12 std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&)
13 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const
------------------------------------------
Python Call Stacks (More useful to users):
------------------------------------------
File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/framework.py", line 2610, in append_op
attrs=kwargs.get("attrs", None))
File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op
return self.main_program.current_block().append_op(*args, **kwargs)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/layers/detection.py", line 2844, in generate_proposals
'RpnRoisLod': rpn_rois_lod
File "/home/aistudio/PaddleDetection/ppdet/core/workspace.py", line 164, in partial_apply
return op(*args, **kwargs_)
File "/home/aistudio/PaddleDetection/ppdet/modeling/anchor_heads/rpn_head.py", line 438, in _get_single_proposals
variances=self.anchor_var)
File "/home/aistudio/PaddleDetection/ppdet/modeling/anchor_heads/rpn_head.py", line 462, in get_proposals
fpn_feat, im_info, lvl, mode)
File "/home/aistudio/PaddleDetection/ppdet/modeling/architectures/cascade_rcnn_cls_aware.py", line 98, in build
rpn_rois = self.rpn_head.get_proposals(body_feats, im_info, mode=mode)
File "/home/aistudio/PaddleDetection/ppdet/modeling/architectures/cascade_rcnn_cls_aware.py", line 315, in train
return self.build(feed_vars, 'train')
File "tools/train.py", line 114, in main
train_fetches = model.train(feed_vars)
File "tools/train.py", line 368, in <module>
main()
----------------------
Error Message Summary:
----------------------
Error: The index of gather_op should not be empty when the index's rank is 1.
[Hint: Expected index.dims()[0] > 0, but received index.dims()[0]:0 <= 0:0.] at (/paddle/paddle/fluid/operators/gather.cu.h:82)
[operator < generate_proposals > error]
terminate called without an active exception
W0805 23:27:24.583714 2619 init.cc:216] Warning: PaddlePaddle catches a failure signal, it may not work properly
W0805 23:27:24.583756 2619 init.cc:218] You could check whether you killed PaddlePaddle thread/process accidentally or report the case to PaddlePaddle
W0805 23:27:24.583762 2619 init.cc:221] The detail failure signal is:
W0805 23:27:24.583772 2619 init.cc:224] *** Aborted at 1596641244 (unix time) try "date -d @1596641244" if you are using GNU date ***
W0805 23:27:24.586570 2619 init.cc:224] PC: @ 0x0 (unknown)
W0805 23:27:24.586745 2619 init.cc:224] *** SIGABRT (@0x3e800000a01) received by PID 2561 (TID 0x7f45ea302700) from PID 2561; stack trace: ***
W0805 23:27:24.588701 2619 init.cc:224] @ 0x7f46172eb390 (unknown)
W0805 23:27:24.590517 2619 init.cc:224] @ 0x7f4616f45428 gsignal
W0805 23:27:24.592346 2619 init.cc:224] @ 0x7f4616f4702a abort
W0805 23:27:24.593598 2619 init.cc:224] @ 0x7f45d7cbd84a __gnu_cxx::__verbose_terminate_handler()
W0805 23:27:24.594558 2619 init.cc:224] @ 0x7f45d7cbbf47 __cxxabiv1::__terminate()
W0805 23:27:24.595700 2619 init.cc:224] @ 0x7f45d7cbbf7d std::terminate()
W0805 23:27:24.596740 2619 init.cc:224] @ 0x7f45d7cbbc5a __gxx_personality_v0
W0805 23:27:24.597666 2619 init.cc:224] @ 0x7f45d7feeb97 _Unwind_ForcedUnwind_Phase2
W0805 23:27:24.598574 2619 init.cc:224] @ 0x7f45d7feee7d _Unwind_ForcedUnwind
W0805 23:27:24.600347 2619 init.cc:224] @ 0x7f46172ea070 __GI___pthread_unwind
W0805 23:27:24.602119 2619 init.cc:224] @ 0x7f46172e2845 __pthread_exit
W0805 23:27:24.602607 2619 init.cc:224] @ 0x557f48b03e59 PyThread_exit_thread
W0805 23:27:24.602751 2619 init.cc:224] @ 0x557f48989c17 PyEval_RestoreThread.cold.798
W0805 23:27:24.604970 2619 init.cc:224] @ 0x7f459e9da959 pybind11::gil_scoped_release::~gil_scoped_release()
W0805 23:27:24.605530 2619 init.cc:224] @ 0x7f459eac2ed5 _ZZN8pybind1112cpp_function10initializeIZN6paddle6pybind10BindReaderEPNS_6moduleEEUlRNS2_9operators6reader40OrderedMultiDeviceLoDTensorBlockingQueueERKSt6vectorINS2_9framework9LoDTensorESaISC_EEE2_bIS9_SG_EINS_4nameENS_9is_methodENS_7siblingENS_10call_guardIINS_18gil_scoped_releaseEEEEEEEvOT_PFT0_DpT1_EDpRKT2_ENUlRNS_6detail13function_callEE1_4_FUNES11_
W0805 23:27:24.607666 2619 init.cc:224] @ 0x7f459e9f8139 pybind11::cpp_function::dispatcher()
W0805 23:27:24.608208 2619 init.cc:224] @ 0x557f48a85744 _PyMethodDef_RawFastCallKeywords
W0805 23:27:24.608693 2619 init.cc:224] @ 0x557f48a85861 _PyCFunction_FastCallKeywords
W0805 23:27:24.609167 2619 init.cc:224] @ 0x557f48af16e8 _PyEval_EvalFrameDefault
W0805 23:27:24.609633 2619 init.cc:224] @ 0x557f48a3581a _PyEval_EvalCodeWithName
W0805 23:27:24.610077 2619 init.cc:224] @ 0x557f48a36635 _PyFunction_FastCallDict
W0805 23:27:24.610554 2619 init.cc:224] @ 0x557f48aee232 _PyEval_EvalFrameDefault
W0805 23:27:24.610965 2619 init.cc:224] @ 0x557f48a84ccb _PyFunction_FastCallKeywords
W0805 23:27:24.611433 2619 init.cc:224] @ 0x557f48aeca93 _PyEval_EvalFrameDefault
W0805 23:27:24.611847 2619 init.cc:224] @ 0x557f48a84ccb _PyFunction_FastCallKeywords
W0805 23:27:24.612315 2619 init.cc:224] @ 0x557f48aeca93 _PyEval_EvalFrameDefault
W0805 23:27:24.612758 2619 init.cc:224] @ 0x557f48a3656b _PyFunction_FastCallDict
W0805 23:27:24.613186 2619 init.cc:224] @ 0x557f48a54e53 _PyObject_Call_Prepend
W0805 23:27:24.613682 2619 init.cc:224] @ 0x557f48a47dbe PyObject_Call
W0805 23:27:24.613884 2619 init.cc:224] @ 0x557f48b44817 t_bootstrap
W0805 23:27:24.613982 2619 init.cc:224] @ 0x557f48aff788 pythread_wrapper
W0805 23:27:24.616046 2619 init.cc:224] @ 0x7f46172e16ba start_thread
Aborted (core dumped)