未验证 提交 15e59c31 编写于 作者: C Chen Weihang 提交者: GitHub

cleanup mmap files in unittest, test=develop (#22814)

上级 89cfa491
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
import sys import sys
import unittest import unittest
import multiprocessing
import numpy as np import numpy as np
import paddle.fluid as fluid import paddle.fluid as fluid
from paddle.fluid import core from paddle.fluid import core
...@@ -50,6 +51,14 @@ class TestDygraphDataLoaderProcess(unittest.TestCase): ...@@ -50,6 +51,14 @@ class TestDygraphDataLoaderProcess(unittest.TestCase):
self.capacity = 2 self.capacity = 2
def test_reader_process_loop(self): def test_reader_process_loop(self):
# This unittest's memory mapped files needs to be cleaned manually
def __clear_process__(util_queue):
while True:
try:
util_queue.get_nowait()
except queue.Empty:
break
with fluid.dygraph.guard(): with fluid.dygraph.guard():
loader = fluid.io.DataLoader.from_generator( loader = fluid.io.DataLoader.from_generator(
capacity=self.batch_num + 1, use_multiprocess=True) capacity=self.batch_num + 1, use_multiprocess=True)
...@@ -58,8 +67,16 @@ class TestDygraphDataLoaderProcess(unittest.TestCase): ...@@ -58,8 +67,16 @@ class TestDygraphDataLoaderProcess(unittest.TestCase):
places=fluid.CPUPlace()) places=fluid.CPUPlace())
loader._data_queue = queue.Queue(self.batch_num + 1) loader._data_queue = queue.Queue(self.batch_num + 1)
loader._reader_process_loop() loader._reader_process_loop()
# For clean memory mapped files
util_queue = multiprocessing.Queue(self.batch_num + 1)
for _ in range(self.batch_num): for _ in range(self.batch_num):
loader._data_queue.get(timeout=10) data = loader._data_queue.get(timeout=10)
util_queue.put(data)
# Clean up memory mapped files
clear_process = multiprocessing.Process(
target=__clear_process__, args=(util_queue, ))
clear_process.start()
def test_reader_process_loop_simple_none(self): def test_reader_process_loop_simple_none(self):
def none_sample_genarator(batch_num): def none_sample_genarator(batch_num):
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册