Error: Blocking queue is killed because the data reader raises an exception
Created by: hsyy673150343
我的数据集部分标签格式: test_image/fz_img_1.jpg [{"transcription": "tibetan", "points": [[224.96774193548384, 351.258064516129], [250.77419354838707, 351.258064516129], [250.77419354838707, 360.2903225806451], [225.61290322580646, 360.2903225806451]]}] test_image/fz_img_10.jpg [{"transcription": "tibetan_1", "points": [[114.0, 491.9032258064516], [420.4516129032258, 516.4193548387096], [418.51612903225805, 565.4516129032257], [111.41935483870967, 538.3548387096774]]}, {"transcription": "tibetan_2", "points": [[1.0967741935483843, 189.96774193548387], [61.741935483870975, 191.25806451612902], [61.741935483870975, 204.16129032258064], [1.0967741935483843, 203.51612903225805]]}, {"transcription": "tibetan_3", "points": [[1.7419354838709609, 216.41935483870967], [64.96774193548387, 218.3548387096774], [64.32258064516128, 232.54838709677418], [1.0967741935483843, 231.25806451612902]]}, {"transcription": "tibetan_4", "points": [[2.3870967741935516, 245.4516129032258], [63.03225806451613, 248.03225806451613], [63.03225806451613, 259.0], [2.3870967741935516, 257.06451612903226]]}] test_image/fz_img_11.jpg [{"transcription": "tibetan_1", "points": [[26.258064516129025, 266.09677419354836], [137.8709677419355, 265.4516129032258], [139.16129032258064, 277.7096774193548], [26.903225806451616, 277.06451612903226]]}, {"transcription": "tibetan_2", "points": [[337.8709677419355, 408.0322580645161], [415.93548387096774, 409.96774193548384], [416.5806451612903, 434.48387096774195], [337.8709677419355, 431.9032258064516]]}] test_image/fz_img_12.jpg [{"transcription": "tibetan_1", "points": [[137.8709677419355, 62.87096774193548], [203.67741935483872, 62.87096774193548], [204.32258064516128, 75.12903225806451], [138.51612903225805, 75.12903225806451]]}, {"transcription": "tibetan_2", "points": [[246.25806451612902, 62.87096774193548], [317.22580645161287, 60.93548387096774], [317.8709677419355, 73.83870967741936], [246.9032258064516, 73.83870967741936]]}, {"transcription": "tibetan_3", "points": [[48.83870967741936, 499.0], [448.8387096774194, 555.1290322580645], [444.3225806451612, 605.4516129032257], [49.48387096774192, 553.1935483870967]]}] test_image/fz_img_13.jpg [{"transcription": "tibetan_1", "points": [[115.29032258064515, 402.22580645161287], [320.4516129032258, 352.5483870967742], [321.741935483871, 368.6774193548387], [116.5806451612903, 416.41935483870964]]}, {"transcription": "tibetan_2", "points": [[326.258064516129, 351.9032258064516], [484.9677419354839, 329.96774193548384], [485.61290322580646, 343.51612903225805], [326.258064516129, 366.741935483871]]}, {"transcription": "tibetan_3", "points": [[423.0322580645161, 251.25806451612902], [492.0645161290322, 284.80645161290323], [486.258064516129, 307.38709677419354], [421.09677419354836, 271.9032258064516]]}] test_image/fz_img_14.jpg [{"transcription": "tibetan", "points": [[160.4516129032258, 398.3548387096774], [444.9677419354839, 466.741935483871], [444.9677419354839, 509.3225806451613], [161.74193548387095, 435.1290322580645]]}] test_image/fz_img_15.jpg [{"transcription": "tibetan", "points": [[137.8709677419355, 515.1290322580645], [514.0, 499.64516129032256], [513.3548387096774, 537.7096774193549], [139.80645161290323, 555.1290322580645]]}] test_image/fz_img_16.jpg [{"transcription": "tibetan", "points": [[156.5806451612903, 527.3870967741935], [518.516129032258, 509.96774193548384], [517.8709677419355, 541.5806451612904], [155.29032258064515, 558.3548387096774]]}]
我使用自己的数据集训练DB模型,出现下面的错误: Process Process-1: Traceback (most recent call last): File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 93, in run self._target(*self._args, **self._kwargs) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 556, in _read_into_queue six.reraise(*sys.exc_info()) File "/home/hs/.local/lib/python3.6/site-packages/six.py", line 703, in reraise raise value File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 549, in _read_into_queue for sample in reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 59, in batch_iter_reader for outs in sample_iter_reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 52, in sample_iter_reader outs = self.process(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 102, in call img_path, gt_label = self.convert_label_infor(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 98, in convert_label_infor label = json.loads(substr[1]) IndexError: list index out of range Process Process-2: Traceback (most recent call last): File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 93, in run self._target(*self._args, **self._kwargs) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 556, in _read_into_queue six.reraise(*sys.exc_info()) File "/home/hs/.local/lib/python3.6/site-packages/six.py", line 703, in reraise raise value File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 549, in _read_into_queue for sample in reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 59, in batch_iter_reader for outs in sample_iter_reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 52, in sample_iter_reader outs = self.process(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 102, in call img_path, gt_label = self.convert_label_infor(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 98, in convert_label_infor label = json.loads(substr[1]) IndexError: list index out of range Process Process-3: Traceback (most recent call last): File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 93, in run self._target(*self._args, **self._kwargs) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 556, in _read_into_queue six.reraise(*sys.exc_info()) File "/home/hs/.local/lib/python3.6/site-packages/six.py", line 703, in reraise raise value File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 549, in _read_into_queue for sample in reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 59, in batch_iter_reader for outs in sample_iter_reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 52, in sample_iter_reader outs = self.process(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 102, in call img_path, gt_label = self.convert_label_infor(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 98, in convert_label_infor label = json.loads(substr[1]) IndexError: list index out of range Process Process-4: Traceback (most recent call last): File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 93, in run self._target(*self._args, **self._kwargs) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 556, in _read_into_queue six.reraise(*sys.exc_info()) File "/home/hs/.local/lib/python3.6/site-packages/six.py", line 703, in reraise raise value File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 549, in _read_into_queue for sample in reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 59, in batch_iter_reader for outs in sample_iter_reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 52, in sample_iter_reader outs = self.process(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 102, in call img_path, gt_label = self.convert_label_infor(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 98, in convert_label_infor label = json.loads(substr[1]) IndexError: list index out of range Process Process-5: Traceback (most recent call last): File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 93, in run self._target(*self._args, **self._kwargs) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 556, in _read_into_queue six.reraise(*sys.exc_info()) File "/home/hs/.local/lib/python3.6/site-packages/six.py", line 703, in reraise raise value File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 549, in _read_into_queue for sample in reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 59, in batch_iter_reader for outs in sample_iter_reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 52, in sample_iter_reader outs = self.process(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 102, in call img_path, gt_label = self.convert_label_infor(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 98, in convert_label_infor label = json.loads(substr[1]) IndexError: list index out of range Process Process-6: Traceback (most recent call last): File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 93, in run self._target(*self._args, **self._kwargs) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 556, in _read_into_queue six.reraise(*sys.exc_info()) File "/home/hs/.local/lib/python3.6/site-packages/six.py", line 703, in reraise raise value File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 549, in _read_into_queue for sample in reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 59, in batch_iter_reader for outs in sample_iter_reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 52, in sample_iter_reader outs = self.process(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 102, in call img_path, gt_label = self.convert_label_infor(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 98, in convert_label_infor label = json.loads(substr[1]) IndexError: list index out of range Process Process-7: Traceback (most recent call last): File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 93, in run self._target(*self._args, **self._kwargs) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 556, in _read_into_queue six.reraise(*sys.exc_info()) File "/home/hs/.local/lib/python3.6/site-packages/six.py", line 703, in reraise raise value File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 549, in _read_into_queue for sample in reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 59, in batch_iter_reader for outs in sample_iter_reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 52, in sample_iter_reader outs = self.process(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 102, in call img_path, gt_label = self.convert_label_infor(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 98, in convert_label_infor label = json.loads(substr[1]) IndexError: list index out of range 2020-06-18 17:20:46,460-WARNING: Your reader has raised an exception! Exception in thread Thread-1: Traceback (most recent call last): File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/threading.py", line 916, in _bootstrap_inner self.run() File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/threading.py", line 864, in run self._target(*self._args, **self._kwargs) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/fluid/reader.py", line 805, in thread_main six.reraise(*sys.exc_info()) File "/home/hs/.local/lib/python3.6/site-packages/six.py", line 703, in reraise raise value File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/fluid/reader.py", line 785, in thread_main for tensors in self._tensor_reader(): File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/fluid/reader.py", line 853, in tensor_reader_impl for slots in paddle_reader(): File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/fluid/data_feeder.py", line 488, in reader_creator for item in reader(): File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 572, in queue_reader raise ValueError("multiprocess reader raises an exception") ValueError: multiprocess reader raises an exception
Process Process-8: Traceback (most recent call last): File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/multiprocessing/process.py", line 93, in run self._target(*self._args, **self._kwargs) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 556, in _read_into_queue six.reraise(*sys.exc_info()) File "/home/hs/.local/lib/python3.6/site-packages/six.py", line 703, in reraise raise value File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/reader/decorator.py", line 549, in _read_into_queue for sample in reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 59, in batch_iter_reader for outs in sample_iter_reader(): File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/dataset_traversal.py", line 52, in sample_iter_reader outs = self.process(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 102, in call img_path, gt_label = self.convert_label_infor(label_infor) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/data/det/db_process.py", line 98, in convert_label_infor label = json.loads(substr[1]) IndexError: list index out of range Traceback (most recent call last): File "tools/train.py", line 114, in main() File "tools/train.py", line 106, in main program.train_eval_det_run(config, exe, train_info_dict, eval_info_dict) File "/home/hs/OCR_Projects/PaddleOCR/tools/program.py", line 238, in train_eval_det_run return_numpy=False) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/fluid/executor.py", line 790, in run six.reraise(*sys.exc_info()) File "/home/hs/.local/lib/python3.6/site-packages/six.py", line 703, in reraise raise value File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/fluid/executor.py", line 785, in run use_program_cache=use_program_cache) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/fluid/executor.py", line 850, in _run_impl return_numpy=return_numpy) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/fluid/executor.py", line 684, in _run_parallel tensors = exe.run(fetch_var_names)._move_to_list() paddle.fluid.core_avx.EnforceNotMet:
C++ Call Stacks (More useful to developers):
0 std::string paddle::platform::GetTraceBackString<std::string const&>(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::operators::reader::BlockingQueue<std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor > >::Receive(std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor >) 3 paddle::operators::reader::PyReader::ReadNext(std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor >) 4 std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result, std::__future_base::_Result_base::_Deleter>, unsigned long> >::_M_invoke(std::_Any_data const&) 5 std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&) 6 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const
Python Call Stacks (More useful to users):
File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/fluid/framework.py", line 2525, in append_op attrs=kwargs.get("attrs", None)) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/fluid/reader.py", line 733, in _init_non_iterable outputs={'Out': self._feed_list}) File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/fluid/reader.py", line 646, in init self._init_non_iterable() File "/home/hs/anaconda3/envs/Paddle_OCR/lib/python3.6/site-packages/paddle/fluid/reader.py", line 280, in from_generator iterable, return_list) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/modeling/architectures/det_model.py", line 97, in create_feed iterable=False) File "/home/hs/OCR_Projects/PaddleOCR/ppocr/modeling/architectures/det_model.py", line 110, in call image, labels, loader = self.create_feed(mode) File "/home/hs/OCR_Projects/PaddleOCR/tools/program.py", line 169, in build dataloader, outputs = model(mode=mode) File "tools/train.py", line 68, in main config, train_program, startup_program, mode='train') File "tools/train.py", line 114, in main()
Error Message Summary:
Error: Blocking queue is killed because the data reader raises an exception [Hint: Expected killed_ != true, but received killed_:1 == true:1.] at (/paddle/paddle/fluid/operators/reader/blocking_queue.h:141) [operator < read > error]