未验证 提交 118aee6f 编写于 作者: Z zhangbo9674 提交者: GitHub

not use shm cache default (#50089)

上级 c64296bf
...@@ -1198,11 +1198,11 @@ PADDLE_DEFINE_EXPORTED_bool(trt_ibuilder_cache, ...@@ -1198,11 +1198,11 @@ PADDLE_DEFINE_EXPORTED_bool(trt_ibuilder_cache,
* mmap_allocator related FLAG * mmap_allocator related FLAG
* Name: use_shm_cache * Name: use_shm_cache
* Since Version: 2.5.0 * Since Version: 2.5.0
* Value Range: bool, default=true * Value Range: bool, default=false
* Example: * Example:
* Note: . If True, mmap_allocator will cache shm file to decrease munmap * Note: . If True, mmap_allocator will cache shm file to decrease munmap
* operation. * operation.
*/ */
PADDLE_DEFINE_EXPORTED_bool(use_shm_cache, PADDLE_DEFINE_EXPORTED_bool(use_shm_cache,
true, false,
"Use shm cache in mmap_allocator."); "Use shm cache in mmap_allocator.");
...@@ -410,6 +410,13 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): ...@@ -410,6 +410,13 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase):
# Note(zhangbo): shm_buffer_size is used for MemoryMapAllocationPool. # Note(zhangbo): shm_buffer_size is used for MemoryMapAllocationPool.
# MemoryMapAllocationPool is used to cache and reuse shm, thus reducing munmap in dataloader. # MemoryMapAllocationPool is used to cache and reuse shm, thus reducing munmap in dataloader.
# For more details, please see: paddle/fluid/memory/allocation/mmap_allocator.h # For more details, please see: paddle/fluid/memory/allocation/mmap_allocator.h
if os.environ.get('FLAGS_use_shm_cache', False) in [
1,
'1',
True,
'True',
'true',
]:
try: try:
self._worker_shm_buffer_size = (2 + 1) * len(self._dataset[0]) self._worker_shm_buffer_size = (2 + 1) * len(self._dataset[0])
except: except:
...@@ -417,6 +424,8 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): ...@@ -417,6 +424,8 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase):
warnings.warn( warnings.warn(
"Setting the shm cache buffer size to 0, equivalent to not using the shm cache policy." "Setting the shm cache buffer size to 0, equivalent to not using the shm cache policy."
) )
else:
self._worker_shm_buffer_size = 0
self._main_thread_shm_buffer_size = ( self._main_thread_shm_buffer_size = (
(self._worker_shm_buffer_size) * 2 * self._num_workers (self._worker_shm_buffer_size) * 2 * self._num_workers
) )
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册