Data reader fails after several iterations
Created by: kuke
Traceback (most recent call last):
File "train.py", line 280, in <module>
train(args)
File "train.py", line 221, in train
args.minimum_batch_size)):
File "/home/disk1/liuyibing/paddle_work/models_aishell/fluid/DeepASR/data_utils/async_data_reader.py", line 422, in batch_iterator
3, self._manager)
File "/home/disk1/liuyibing/paddle_work/models_aishell/fluid/DeepASR/data_utils/util.py", line 168, in __init__
self._dict[name] = SharedNDArray(name)
File "<string>", line 2, in __setitem__
File "/home/work/.jumbo/lib/python2.7/multiprocessing/managers.py", line 773, in _callmethod
raise convert_to_error(kind, result)
multiprocessing.managers.RemoteError:
---------------------------------------------------------------------------
Traceback (most recent call last):
File "/home/work/.jumbo/lib/python2.7/multiprocessing/managers.py", line 240, in serve_client
request = recv()
File "/home/disk1/liuyibing/paddle_work/models_aishell/fluid/DeepASR/data_utils/util.py", line 144, in __setstate__
self.zeros_like(state[1], state[2])
File "/home/disk1/liuyibing/paddle_work/models_aishell/fluid/DeepASR/data_utils/util.py", line 110, in zeros_like
self._name, posix_ipc.O_CREAT, size=size)
OSError: This process already has the maximum number of files open
---------------------------------------------------------------------------
Traceback (most recent call last):
File "/home/work/.jumbo/lib/python2.7/multiprocessing/util.py", line 261, in _run_finalizers
File "/home/work/.jumbo/lib/python2.7/multiprocessing/util.py", line 200, in __call__
File "/home/work/.jumbo/lib/python2.7/shutil.py", line 237, in rmtree
File "/home/work/.jumbo/lib/python2.7/shutil.py", line 235, in rmtree
OSError: [Errno 24] Too many open files: '/tmp/pymp-KdbIU1'