Created by: mikeFang1986

系统：ubuntu16.04 CUDA Version 10.0.130 CUDnn 7.4.2 Nccl 2.4.8 python 3.7

python -u tools/train.py -c configs/yolov3_mobilenet_v1_fruit.yml --use_tb=True --tb_log_dir=tb_fruit_dir/scalar --eval 运行时，报错如下： use_gpu: true weights: output/yolov3_mobilenet_v1_fruit/best_model

2019-12-30 17:02:35,740-INFO: places would be ommited when DataLoader is not iterable W1230 17:02:36.778067 9089 device_context.cc:236] Please NOTE: device: 0, CUDA Capability: 61, Driver API Version: 10.1, Runtime API Version: 10.0 W1230 17:02:36.781242 9089 device_context.cc:244] device: 0, cuDNN Version: 7.4. W1230 17:02:36.781275 9089 device_context.cc:270] WARNING: device: 0. The installed Paddle is compiled with CUDNN 7.6, but CUDNN version in your machine is 7.4, which may cause serious incompatible bug. Please recompile or reinstall Paddle with compatible CUDNN version. 2019-12-30 17:02:37,792-INFO: Found /home/css/.cache/paddle/weights/yolov3_mobilenet_v1 2019-12-30 17:02:37,792-INFO: Loading parameters from /home/css/.cache/paddle/weights/yolov3_mobilenet_v1... 2019-12-30 17:02:37,797-INFO: In load_params, ignore yolo_output.0.conv.weights 2019-12-30 17:02:37,797-INFO: In load_params, ignore yolo_output.0.conv.bias 2019-12-30 17:02:37,798-INFO: In load_params, ignore yolo_output.1.conv.weights 2019-12-30 17:02:37,798-INFO: In load_params, ignore yolo_output.1.conv.bias 2019-12-30 17:02:37,800-INFO: In load_params, ignore yolo_output.2.conv.weights 2019-12-30 17:02:37,800-INFO: In load_params, ignore yolo_output.2.conv.bias 2019-12-30 17:02:39,540-INFO: places would be ommited when DataLoader is not iterable I1230 17:02:39.740296 9089 parallel_executor.cc:421] The number of CUDAPlace, which is used in ParallelExecutor, is 1. And the Program will be copied 1 copies I1230 17:02:39.834806 9089 graph_pattern_detector.cc:96] --- detected 1 subgraphs I1230 17:02:39.953145 9089 build_strategy.cc:363] SeqOnlyAllReduceOps:0, num_trainers:1 I1230 17:02:40.066726 9089 parallel_executor.cc:285] Inplace strategy is enabled, when build_strategy.enable_inplace = True I1230 17:02:40.164924 9089 parallel_executor.cc:368] Garbage collection strategy is enabled, when FLAGS_eager_delete_tensor_gb = 0 /home/css/anaconda3/envs/fl_pp_env/lib/python3.7/site-packages/paddle/fluid/executor.py:779: UserWarning: The following exception is not an EOF exception. "The following exception is not an EOF exception.") Traceback (most recent call last): File "tools/train.py", line 340, in main() File "tools/train.py", line 246, in main outs = exe.run(compiled_train_prog, fetch_list=train_values) File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/site-packages/paddle/fluid/executor.py", line 780, in run six.reraise(*sys.exc_info()) File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/site-packages/six.py", line 696, in reraise raise value File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/site-packages/paddle/fluid/executor.py", line 775, in run use_program_cache=use_program_cache) File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/site-packages/paddle/fluid/executor.py", line 834, in _run_impl return_numpy=return_numpy) File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/site-packages/paddle/fluid/executor.py", line 674, in _run_parallel tensors = exe.run(fetch_var_names)._move_to_list() paddle.fluid.core_avx.EnforceNotMet:

---

C++ Call Stacks (More useful to developers):

0 std::string paddle::platform::GetTraceBackStringstd::string(std::string&&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(paddle::platform::ErrorSummary const&, char const*, int) 2 paddle::operators::BatchNormKernel<paddle::platform::CUDADeviceContext, float>::Compute(paddle::framework::ExecutionContext const&) const 3 std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::BatchNormKernel<paddle::platform::CUDADeviceContext, float>, paddle::operators::BatchNormKernel<paddle::platform::CUDADeviceContext, double>, paddle::operators::BatchNormKernel<paddle::platform::CUDADeviceContext, paddle::platform::float16> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1 (closed)}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) 4 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, paddle::platform::Place const&, paddle::framework::RuntimeContext*) const 5 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, paddle::platform::Place const&) const 6 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, paddle::platform::Place const&) 7 paddle::framework::details::ComputationOpHandle::RunImpl() 8 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 9 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr<paddle::framework::BlockingQueue > const&, unsigned long*) 10 std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 11 std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&) 12 ThreadPool::ThreadPool(unsigned long)::{lambda()#1 (closed)}::operator()() const

---

Python Call Stacks (More useful to users):

File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/site-packages/paddle/fluid/framework.py", line 2488, in append_op attrs=kwargs.get("attrs", None)) File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/site-packages/paddle/fluid/layers/nn.py", line 4363, in batch_norm attrs=attrs) File "/home/css/work/fl/pp_env/fast_rcnn/PaddleDetection/PaddleDetection/ppdet/modeling/backbones/mobilenet.py", line 97, in _conv_norm moving_variance_name=bn_name + '_variance') File "/home/css/work/fl/pp_env/fast_rcnn/PaddleDetection/PaddleDetection/ppdet/modeling/backbones/mobilenet.py", line 157, in call input, 3, int(32 * scale), 2, 1, name=self.prefix_name + "conv1") File "/home/css/work/fl/pp_env/fast_rcnn/PaddleDetection/PaddleDetection/ppdet/modeling/architectures/yolov3.py", line 56, in build body_feats = self.backbone(im) File "/home/css/work/fl/pp_env/fast_rcnn/PaddleDetection/PaddleDetection/ppdet/modeling/architectures/yolov3.py", line 80, in train return self.build(feed_vars, mode='train') File "tools/train.py", line 128, in main train_fetches = model.train(feed_vars) File "tools/train.py", line 340, in main()

---

Error Message Summary:

Error: CUDNN_STATUS_BAD_PARAM at (/paddle/paddle/fluid/operators/batch_norm_op.cu:183) [operator < batch_norm > error]

^CError in atexit._run_exitfuncs: Traceback (most recent call last): Process Process-6: File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/multiprocessing/popen_fork.py", line 28, in poll pid, sts = os.waitpid(self.pid, flag) KeyboardInterrupt Traceback (most recent call last): File "/home/css/work/fl/pp_env/fast_rcnn/PaddleDetection/PaddleDetection/ppdet/data/transform/transformer.py", line 44, in _proxy_method return func(*args, **kwargs) File "/home/css/work/fl/pp_env/fast_rcnn/PaddleDetection/PaddleDetection/ppdet/data/dataset.py", line 48, in reset (self.class.name)) NotImplementedError: MappedDataset.reset not available

During handling of the above exception, another exception occurred:

Traceback (most recent call last): File "/home/css/work/fl/pp_env/fast_rcnn/PaddleDetection/PaddleDetection/ppdet/data/transform/transformer.py", line 44, in _proxy_method return func(*args, **kwargs) File "/home/css/work/fl/pp_env/fast_rcnn/PaddleDetection/PaddleDetection/ppdet/data/dataset.py", line 48, in reset (self.class.name)) NotImplementedError: BatchedDataset.reset not available

During handling of the above exception, another exception occurred:

Traceback (most recent call last): File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap self.run() File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/multiprocessing/process.py", line 99, in run self._target(*self._args, **self._kwargs) File "/home/css/work/fl/pp_env/fast_rcnn/PaddleDetection/PaddleDetection/ppdet/data/transform/parallel_map.py", line 153, in _consume outq.put(result) File "/home/css/work/fl/pp_env/fast_rcnn/PaddleDetection/PaddleDetection/ppdet/data/transform/shared_queue/queue.py", line 71, in put super(SharedQueue, self).put(buff, **kwargs) File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/multiprocessing/queues.py", line 89, in put self._notempty.notify() File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/threading.py", line 352, in notify waiter.release() File "/home/css/work/fl/pp_env/fast_rcnn/PaddleDetection/PaddleDetection/ppdet/data/transform/parallel_map.py", line 226, in _reader_exit sys.exit() SystemExit

During handling of the above exception, another exception occurred:

Traceback (most recent call last): File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/multiprocessing/process.py", line 300, in _bootstrap util._exit_function() File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/multiprocessing/util.py", line 337, in _exit_function _run_finalizers() File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/multiprocessing/util.py", line 277, in _run_finalizers finalizer() File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/multiprocessing/util.py", line 201, in call res = self._callback(*self._args, **self._kwargs) File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/multiprocessing/queues.py", line 192, in _finalize_join thread.join() File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/threading.py", line 1044, in join self._wait_for_tstate_lock() File "/home/css/anaconda3/envs/fl_pp_env/lib/python3.7/threading.py", line 1060, in _wait_for_tstate_lock elif lock.acquire(block, timeout): KeyboardInterrupt

python ppdet/modeling/tests/test_architectures.py，确认通过