diff --git a/fluid/DeepASR/data_utils/async_data_reader.py b/fluid/DeepASR/data_utils/async_data_reader.py index f88d309151bdd83a149c9d6e22911ebe1d48ae74..1515b299d4357eac16892dceda6e4f05bf1fc045 100644 --- a/fluid/DeepASR/data_utils/async_data_reader.py +++ b/fluid/DeepASR/data_utils/async_data_reader.py @@ -11,7 +11,6 @@ import time import numpy as np from threading import Thread import signal -import multiprocessing from multiprocessing import Manager, Process import data_utils.augmentor.trans_mean_variance_norm as trans_mean_variance_norm import data_utils.augmentor.trans_add_delta as trans_add_delta diff --git a/fluid/DeepASR/data_utils/util.py b/fluid/DeepASR/data_utils/util.py index 0c8e311ab09077b6ad6bcc264ebaa1fa463978ef..d9022dbc90e5313609be36578337a99930c06c90 100644 --- a/fluid/DeepASR/data_utils/util.py +++ b/fluid/DeepASR/data_utils/util.py @@ -77,6 +77,15 @@ class CriticalException(Exception): class SharedNDArray(object): + """SharedNDArray utilizes shared memory to avoid data serialization when + object of which shared between different processes. We can reconstruct the + ndarray when memory address provided. + + Args: + name (str): Address name of shared memory. + is_verify (bool): Whether to do validation for writing operation. + """ + def __init__(self, name, is_verify=False): self._name = name self._shm = None @@ -130,6 +139,18 @@ class SharedNDArray(object): class SharedMemoryPoolManager(object): + """SharedMemoryPoolManager maintains a multiprocessing.Manager.dict object. + All available addresses are allocated once and will be reused. Though this + class is not process-safe, the pool can be shared between processes. All + shared memory should be unlinked before the main process exited. + + Args: + pool_size (int): Size of shared memory pool. + manager (dict): A multiprocessing.Manager object, the pool is + maintained by the proxy process. + name_prefix (str): Address prefix of shared memory. + """ + def __init__(self, pool_size, manager, name_prefix='/deep_asr'): self._names = [] self._dict = manager.dict()