From 3adaf7f548bb8c3358b7888a1b6a7cb131bfd8ad Mon Sep 17 00:00:00 2001 From: Xinghai Sun Date: Tue, 13 Mar 2018 15:53:03 +0800 Subject: [PATCH] Add comments for new classes. --- fluid/DeepASR/data_utils/async_data_reader.py | 1 - fluid/DeepASR/data_utils/util.py | 21 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/fluid/DeepASR/data_utils/async_data_reader.py b/fluid/DeepASR/data_utils/async_data_reader.py index f88d3091..1515b299 100644 --- a/fluid/DeepASR/data_utils/async_data_reader.py +++ b/fluid/DeepASR/data_utils/async_data_reader.py @@ -11,7 +11,6 @@ import time import numpy as np from threading import Thread import signal -import multiprocessing from multiprocessing import Manager, Process import data_utils.augmentor.trans_mean_variance_norm as trans_mean_variance_norm import data_utils.augmentor.trans_add_delta as trans_add_delta diff --git a/fluid/DeepASR/data_utils/util.py b/fluid/DeepASR/data_utils/util.py index 0c8e311a..d9022dbc 100644 --- a/fluid/DeepASR/data_utils/util.py +++ b/fluid/DeepASR/data_utils/util.py @@ -77,6 +77,15 @@ class CriticalException(Exception): class SharedNDArray(object): + """SharedNDArray utilizes shared memory to avoid data serialization when + object of which shared between different processes. We can reconstruct the + ndarray when memory address provided. + + Args: + name (str): Address name of shared memory. + is_verify (bool): Whether to do validation for writing operation. + """ + def __init__(self, name, is_verify=False): self._name = name self._shm = None @@ -130,6 +139,18 @@ class SharedNDArray(object): class SharedMemoryPoolManager(object): + """SharedMemoryPoolManager maintains a multiprocessing.Manager.dict object. + All available addresses are allocated once and will be reused. Though this + class is not process-safe, the pool can be shared between processes. All + shared memory should be unlinked before the main process exited. + + Args: + pool_size (int): Size of shared memory pool. + manager (dict): A multiprocessing.Manager object, the pool is + maintained by the proxy process. + name_prefix (str): Address prefix of shared memory. + """ + def __init__(self, pool_size, manager, name_prefix='/deep_asr'): self._names = [] self._dict = manager.dict() -- GitLab