Created by: Hizhaoyuan

raceback (most recent call last): File "/temp/icnet_test/train.py", line 153, in main() File "/temp/icnet_test/train.py", line 149, in main train(args) File "/temp/icnet_test/train.py", line 121, in train fetch_list=[reduced_loss, loss_sub4, loss_sub24, loss_sub124]) File "/local/lib/python2.7/site-packages/paddle/fluid/executor.py", line 470, in run self.executor.run(program.desc, scope, 0, True, True) paddle.fluid.core.EnforceNotMet: CUDNN_STATUS_BAD_PARAM at [/paddle/paddle/fluid/operators/batch_norm_op.cu.cc:114] PaddlePaddle Call Stacks: 0 0x7f4b264f3776p paddle::platform::EnforceNotMet::EnforceNotMet(std::exception_ptr::exception_ptr, char const*, int) + 486 1 0x7f4b267733bdp paddle::operators::BatchNormKernel<paddle::platform::CUDADeviceContext, float>::Compute(paddle::framework::ExecutionContext const&) const + 5133 2 0x7f4b26774123p std::Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::BatchNormKernel<paddle::platform::CUDADeviceContext, float>, paddle::operators::BatchNormKernel<paddle::platform::CUDADeviceContext, double>, paddle::operators::BatchNormKernel<paddle::platform::CUDADeviceContext, paddle::platform::float16> >::operator()(char const*, char const*) const::{lambda(paddle::framework::ExecutionContext const&)#1 (closed)}>::M_invoke(std::Any_data const&, paddle::framework::ExecutionContext const&) + 35 3 0x7f4b27b6ec3cp paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 492 4 0x7f4b27b6b47fp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 463 5 0x7f4b265c143bp paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 219 6 0x7f4b265c2090p paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool) + 128 7 0x7f4b264d6a5dp 8 0x7f4b26509e84p pybind11::cpp_function::dispatcher(_object*, _object*, _object*) + 2596 9 0x4c5326p PyEval_EvalFrameEx + 37958 10 0x4b9b66p PyEval_EvalCodeEx + 774 11 0x4c17c6p PyEval_EvalFrameEx + 22758 12 0x4b9b66p PyEval_EvalCodeEx + 774 13 0x4c1f56p PyEval_EvalFrameEx + 24694 14 0x4b9b66p PyEval_EvalCodeEx + 774 15 0x4c1f56p PyEval_EvalFrameEx + 24694 16 0x4b9b66p PyEval_EvalCodeEx + 774 17 0x4eb69fp 18 0x4e58f2p PyRun_FileExFlags + 130 19 0x4e41a6p PyRun_SimpleFileExFlags + 390 20 0x4938cep Py_Main + 1358 21 0x7f4b67c65830p __libc_start_main + 240 22 0x493299p _start + 41