Created by: mingzhanliao

使用了该命令 python -u tools/train.py -c configs/yolov3_mobilenet_v1_fruit.yml
--use_tb=True
--tb_log_dir=tb_fruit_dir/scalar
--eval 020-05-09 18:15:14,362-INFO: iter: 5000, lr: 0.000010, 'loss': '7.509343', time: 0.198, eta: 0:49:35 2020-05-09 18:15:14,363-INFO: Save model to output/yolov3_mobilenet_v1_fruit/5000. 2020-05-09 18:15:17,492-INFO: Test iter 0 2020-05-09 18:15:19,401-INFO: Test finish iter 60 2020-05-09 18:15:19,401-INFO: Total number of images: 60, inference time: 30.967490429494507 fps. 2020-05-09 18:15:19,402-INFO: Start evaluate... 2020-05-09 18:15:19,482-INFO: Accumulating evaluatation results... 2020-05-09 18:15:19,485-INFO: mAP(0.50, 11point) = 49.46 2020-05-09 18:15:19,486-INFO: Best test box ap: 66.73345995843638, in iter: 4400 2020-05-09 18:15:20,705-INFO: iter: 5020, lr: 0.000010, 'loss': '7.508121', time: 0.317, eta: 1:19:15 2020-05-09 18:15:23,712-WARNING: consumer[8475] exit abnormally with exitcode[-9] 2020-05-09 18:15:23,713-WARNING: 1 consumers have exited abnormally!!! 2020-05-09 18:15:23,713-WARNING: consumer[8475] exit abnormally with exitcode[-9] 2020-05-09 18:15:23,713-WARNING: 1 consumers have exited abnormally!!! 2020-05-09 18:15:23,714-WARNING: Your reader has raised an exception! /usr/local/lib/python3.6/dist-packages/paddle/fluid/executor.py:789: UserWarning: The following exception is not an EOF exception. "The following exception is not an EOF exception.") Traceback (most recent call last): File "tools/train.py", line 379, in main() File "tools/train.py", line 252, in main outs = exe.run(compiled_train_prog, fetch_list=train_values) File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/executor.py", line 790, in run six.reraise(*sys.exc_info()) File "/usr/local/lib/python3.6/dist-packages/six.py", line 703, in reraise raise value File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/executor.py", line 785, in run use_program_cache=use_program_cache) File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/executor.py", line 850, in _run_impl return_numpy=return_numpy) File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/executor.py", line 684, in _run_parallel tensors = exe.run(fetch_var_names)._move_to_list() paddle.fluid.core_avx.EnforceNotMet:

C++ Call Stacks (More useful to developers):

0 std::string paddle::platform::GetTraceBackString<std::string const&>(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::reader::BlockingQueue<std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor > >::Receive(std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor >) 3 paddle::operators::reader::PyReader::ReadNext(std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor >) 4 std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result, std::__future_base::_Result_base::_Deleter>, unsigned long> >::_M_invoke(std::_Any_data const&) 5 std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&) 6 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const

Python Call Stacks (More useful to users):

File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/framework.py", line 2525, in append_op attrs=kwargs.get("attrs", None)) File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/reader.py", line 733, in _init_non_iterable outputs={'Out': self._feed_list}) File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/reader.py", line 646, in init self._init_non_iterable() File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/reader.py", line 280, in from_generator iterable, return_list) File "/root/PaddleDetection/ppdet/modeling/architectures/yolo.py", line 152, in build_inputs iterable=iterable) if use_dataloader else None File "tools/train.py", line 125, in main feed_vars, train_loader = model.build_inputs(**inputs_def) File "tools/train.py", line 379, in main()

Error Message Summary:

Error: Blocking queue is killed because the data reader raises an exception [Hint: Expected killed_ != true, but received killed_:1 == true:1.] at (/paddle/paddle/fluid/operators/reader/blocking_queue.h:141) [operator < read > error]Exception in thread Thread-11: Traceback (most recent call last): File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner self.run() File "/usr/lib/python3.6/threading.py", line 864, in run self._target(*self._args, **self._kwargs) File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/reader.py", line 805, in thread_main six.reraise(*sys.exc_info()) File "/usr/local/lib/python3.6/dist-packages/six.py", line 703, in reraise raise value File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/reader.py", line 785, in thread_main for tensors in self._tensor_reader(): File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/reader.py", line 853, in tensor_reader_impl for slots in paddle_reader(): File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/data_feeder.py", line 488, in reader_creator for item in reader(): File "/root/PaddleDetection/ppdet/data/reader.py", line 421, in _reader reader.reset() File "/root/PaddleDetection/ppdet/data/parallel_map.py", line 267, in reset assert self._consumer_healthy(), "cannot start another pass of data"
AssertionError: cannot start another pass of data for some consumers exited abnormally before!!!