Created by: mozpp

docker+Paddle+YOLOv3。Paddle为：latest-gpu-cuda9.0-cudnn7。 应该是cudnn的问题，请问怎么解决

ssh://root@127.0.0.1:8022/usr/bin/python -u /project/PaddleCV/yolov3/infer.py --data_dir=/dataset/coco --dataset=coco2017 --weights=./weights/yolov3 --image_path=/dataset/coco/val2017/ --image_name=000000000139.jpg ----------- Configuration Arguments ----------- batch_size: 8 class_num: 80 data_dir: /dataset/coco dataset: coco2017 debug: False draw_thresh: 0.5 image_name: 000000000139.jpg image_path: /dataset/coco/val2017/ input_size: 608 label_smooth: True learning_rate: 0.001 max_iter: 500200 model_save_dir: checkpoints nms_posk: 100 nms_thresh: 0.45 nms_topk: 400 no_mixup_iter: 40000 pretrain: weights/darknet53 random_shape: True snapshot_iter: 2000 start_iter: 0 use_gpu: True use_multiprocess: True valid_thresh: 0.005 weights: ./weights/yolov3

W0726 03:05:00.758013 994 device_context.cc:259] Please NOTE: device: 0, CUDA Capability: 75, Driver API Version: 10.1, Runtime API Version: 9.0 W0726 03:05:00.760088 994 device_context.cc:267] device: 0, cuDNN Version: 7.4. loading annotations into memory... Done (t=0.39s) creating index... index created! Load in 80 categories. An exception was thrown! Invoke operator conv2d error. Python Callstacks: File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/framework.py", line 1773, in append_op attrs=kwargs.get("attrs", None)) File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(args, kwargs) File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/layers/nn.py", line 2194, in conv2d 'fuse_relu_before_depthwise_conv': False File "/project/PaddleCV/yolov3/models/darknet.py", line 37, in conv_bn_layer bias_attr=False) File "/project/PaddleCV/yolov3/models/darknet.py", line 98, in add_DarkNet53_conv_body name="yolo_input") File "/project/PaddleCV/yolov3/models/yolov3.py", line 110, in build_model blocks = add_DarkNet53_conv_body(self.image, not self.is_train) File "/project/PaddleCV/yolov3/infer.py", line 36, in infer model.build_model() File "/project/PaddleCV/yolov3/infer.py", line 85, in infer() C++ Callstacks: CUDNN_STATUS_EXECUTION_FAILED at [/paddle/paddle/fluid/operators/conv_cudnn_op.cu.cc:145] PaddlePaddle Call Stacks: 0 0x7f6754be5978p void paddle::platform::EnforceNotMet::Initstd::string(std::string, char const, int) + 360 1 0x7f6754be5cc7p paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const, int) + 87 2 0x7f67550ece9ep paddle::operators::CUDNNConvOpKernel::Compute(paddle::framework::ExecutionContext const&) const + 2382 3 0x7f67550ed333p std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::CUDNNConvOpKernel, paddle::operators::CUDNNConvOpKernel, paddle::operators::CUDNNConvOpKernelpaddle::platform::float16 >::operator()(char const, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1 (closed)}>::M_invoke(std::Any_data const&, paddle::framework::ExecutionContext const&) + 35 4 0x7f6756c3a5abp paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::framework::RuntimeContext*) const + 411 5 0x7f6756c3aba1p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 529 6 0x7f6756c37f1cp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 332 7 0x7f6754d71f1ep paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 398 8 0x7f6754d74f6fp paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, std::vector<std::string, std::allocatorstd::string > const&, bool) + 143 9 0x7f6754bd6b3dp 10 0x7f6754c182c6p 11 0x4c5326p PyEval_EvalFrameEx + 37958 12 0x4b9b66p PyEval_EvalCodeEx + 774 13 0x4c1f56p PyEval_EvalFrameEx + 24694 14 0x4b9b66p PyEval_EvalCodeEx + 774 15 0x4c1f56p PyEval_EvalFrameEx + 24694 16 0x4b9b66p PyEval_EvalCodeEx + 774 17 0x4c17c6p PyEval_EvalFrameEx + 22758 18 0x4c141fp PyEval_EvalFrameEx + 21823 19 0x4b9b66p PyEval_EvalCodeEx + 774 20 0x4eb69fp 21 0x4e58f2p PyRun_FileExFlags + 130 22 0x4e41a6p PyRun_SimpleFileExFlags + 390 23 0x4938cep Py_Main + 1358 24 0x7f67b9199830p __libc_start_main + 240 25 0x493299p _start + 41

Traceback (most recent call last): File "/project/PaddleCV/yolov3/infer.py", line 85, in infer() File "/project/PaddleCV/yolov3/infer.py", line 69, in infer return_numpy=False) File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/executor.py", line 644, in run raise e paddle.fluid.core_avx.EnforceNotMet: Invoke operator conv2d error. Python Callstacks: File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/framework.py", line 1773, in append_op attrs=kwargs.get("attrs", None)) File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(args, kwargs) File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/layers/nn.py", line 2194, in conv2d 'fuse_relu_before_depthwise_conv': False File "/project/PaddleCV/yolov3/models/darknet.py", line 37, in conv_bn_layer bias_attr=False) File "/project/PaddleCV/yolov3/models/darknet.py", line 98, in add_DarkNet53_conv_body name="yolo_input") File "/project/PaddleCV/yolov3/models/yolov3.py", line 110, in build_model blocks = add_DarkNet53_conv_body(self.image, not self.is_train) File "/project/PaddleCV/yolov3/infer.py", line 36, in infer model.build_model() File "/project/PaddleCV/yolov3/infer.py", line 85, in infer() C++ Callstacks: CUDNN_STATUS_EXECUTION_FAILED at [/paddle/paddle/fluid/operators/conv_cudnn_op.cu.cc:145] PaddlePaddle Call Stacks: 0 0x7f6754be5978p void paddle::platform::EnforceNotMet::Initstd::string(std::string, char const, int) + 360 1 0x7f6754be5cc7p paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const, int) + 87 2 0x7f67550ece9ep paddle::operators::CUDNNConvOpKernel::Compute(paddle::framework::ExecutionContext const&) const + 2382 3 0x7f67550ed333p std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::CUDNNConvOpKernel, paddle::operators::CUDNNConvOpKernel, paddle::operators::CUDNNConvOpKernelpaddle::platform::float16 >::operator()(char const, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1 (closed)}>::M_invoke(std::Any_data const&, paddle::framework::ExecutionContext const&) + 35 4 0x7f6756c3a5abp paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::framework::RuntimeContext*) const + 411 5 0x7f6756c3aba1p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 529 6 0x7f6756c37f1cp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 332 7 0x7f6754d71f1ep paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 398 8 0x7f6754d74f6fp paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, std::vector<std::string, std::allocatorstd::string > const&, bool) + 143 9 0x7f6754bd6b3dp 10 0x7f6754c182c6p 11 0x4c5326p PyEval_EvalFrameEx + 37958 12 0x4b9b66p PyEval_EvalCodeEx + 774 13 0x4c1f56p PyEval_EvalFrameEx + 24694 14 0x4b9b66p PyEval_EvalCodeEx + 774 15 0x4c1f56p PyEval_EvalFrameEx + 24694 16 0x4b9b66p PyEval_EvalCodeEx + 774 17 0x4c17c6p PyEval_EvalFrameEx + 22758 18 0x4c141fp PyEval_EvalFrameEx + 21823 19 0x4b9b66p PyEval_EvalCodeEx + 774 20 0x4eb69fp 21 0x4e58f2p PyRun_FileExFlags + 130 22 0x4e41a6p PyRun_SimpleFileExFlags + 390 23 0x4938cep Py_Main + 1358 24 0x7f67b9199830p __libc_start_main + 240 25 0x493299p _start + 41

Process finished with exit code 1