Created by: phamkhactu

read some image failed

Process Process-1:
Traceback (most recent call last):
  File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
    self.run()
  File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/lib/python3.6/dist-packages/paddle/reader/decorator.py", line 556, in _read_into_queue
    six.reraise(*sys.exc_info())
  File "/root/.local/lib/python3.6/site-packages/six.py", line 703, in reraise
    raise value
  File "/usr/local/lib/python3.6/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
    for sample in reader():
  File "/root/phamkhactu/face_detection/reader.py", line 270, in reader
    image_path)
  File "/root/phamkhactu/face_detection/reader.py", line 119, in preprocess
    settings.min_face_size)
  File "/root/phamkhactu/face_detection/image_util.py", line 437, in crop_image_sampling
    sample_img = np.zeros((height, width, 3))
MemoryError
2020-03-02 15:47:38,957-WARNING: Your decorated reader has raised an exception!
Exception in thread Thread-1:
Traceback (most recent call last):
  File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.6/threading.py", line 864, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/layers/io.py", line 491, in __provider_thread__
    six.reraise(*sys.exc_info())
  File "/root/.local/lib/python3.6/site-packages/six.py", line 703, in reraise
    raise value
  File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/layers/io.py", line 472, in __provider_thread__
    for tensors in func():
  File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/layers/io.py", line 523, in __tensor_provider__
    for slots in paddle_reader():
  File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/data_feeder.py", line 488, in __reader_creator__
    for item in reader():
  File "/usr/local/lib/python3.6/dist-packages/paddle/reader/decorator.py", line 572, in queue_reader
    raise ValueError("multiprocess reader raises an exception")
ValueError: multiprocess reader raises an exception

Pass 0, batch 680, face loss 4.203463, head loss 4.551610, time 0.90225
/usr/local/lib/python3.6/dist-packages/paddle/fluid/executor.py:782: UserWarning: The following exception is not an EOF exception.
  "The following exception is not an EOF exception.")
Traceback (most recent call last):
  File "train.py", line 284, in <module>
    train(args, config, train_parameters, train_file_list)
  File "train.py", line 212, in train
    [v.name for v in fetches])
  File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/parallel_executor.py", line 311, in run
    return_numpy=return_numpy)
  File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/executor.py", line 783, in run
    six.reraise(*sys.exc_info())
  File "/root/.local/lib/python3.6/site-packages/six.py", line 703, in reraise
    raise value
  File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/executor.py", line 778, in run
    use_program_cache=use_program_cache)
  File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/executor.py", line 843, in _run_impl
    return_numpy=return_numpy)
  File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/executor.py", line 677, in _run_parallel
    tensors = exe.run(fetch_var_names)._move_to_list()
paddle.fluid.core_avx.EnforceNotMet: 

--------------------------------------------
C++ Call Stacks (More useful to developers):
--------------------------------------------
0   std::string paddle::platform::GetTraceBackString<std::string const&>(std::string const&, char const*, int)
1   paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int)
2   paddle::operators::reader::BlockingQueue<std::vector<paddle::framework::LoDTensor, std::allocator<paddle::framework::LoDTensor> > >::Receive(std::vector<paddle::framework::LoDTensor, std::allocator<paddle::framework::LoDTensor> >*)
3   paddle::operators::reader::PyReader::ReadNext(std::vector<paddle::framework::LoDTensor, std::allocator<paddle::framework::LoDTensor> >*)
4   std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<unsigned long>, std::__future_base::_Result_base::_Deleter>, unsigned long> >::_M_invoke(std::_Any_data const&)
5   std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&)
6   ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const

------------------------------------------
Python Call Stacks (More useful to users):
------------------------------------------
  File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/framework.py", line 2525, in append_op
    attrs=kwargs.get("attrs", None))
  File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/layer_helper.py", line 43, in append_op
    return self.main_program.current_block().append_op(*args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/paddle/fluid/layers/io.py", line 889, in read_file
    type='read', inputs={'Reader': [reader]}, outputs={'Out': out})
  File "train.py", line 101, in build_program
    image, face_box, head_box, gt_label = fluid.layers.read_file(py_reader)
  File "train.py", line 153, in train
    args=args)
  File "train.py", line 284, in <module>
    train(args, config, train_parameters, train_file_list)

----------------------
Error Message Summary:
----------------------
Error: Blocking queue is killed because the data reader raises an exception
  [Hint: Expected killed_ != true, but received killed_:1 == true:1.] at (/paddle/paddle/fluid/operators/reader/blocking_queue.h:141)
  [operator < read > error]