未验证 提交 118aee6f 编写于 作者: Z zhangbo9674 提交者: GitHub

not use shm cache default (#50089)

上级 c64296bf
...@@ -1198,11 +1198,11 @@ PADDLE_DEFINE_EXPORTED_bool(trt_ibuilder_cache, ...@@ -1198,11 +1198,11 @@ PADDLE_DEFINE_EXPORTED_bool(trt_ibuilder_cache,
* mmap_allocator related FLAG * mmap_allocator related FLAG
* Name: use_shm_cache * Name: use_shm_cache
* Since Version: 2.5.0 * Since Version: 2.5.0
* Value Range: bool, default=true * Value Range: bool, default=false
* Example: * Example:
* Note: . If True, mmap_allocator will cache shm file to decrease munmap * Note: . If True, mmap_allocator will cache shm file to decrease munmap
* operation. * operation.
*/ */
PADDLE_DEFINE_EXPORTED_bool(use_shm_cache, PADDLE_DEFINE_EXPORTED_bool(use_shm_cache,
true, false,
"Use shm cache in mmap_allocator."); "Use shm cache in mmap_allocator.");
...@@ -410,13 +410,22 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): ...@@ -410,13 +410,22 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase):
# Note(zhangbo): shm_buffer_size is used for MemoryMapAllocationPool. # Note(zhangbo): shm_buffer_size is used for MemoryMapAllocationPool.
# MemoryMapAllocationPool is used to cache and reuse shm, thus reducing munmap in dataloader. # MemoryMapAllocationPool is used to cache and reuse shm, thus reducing munmap in dataloader.
# For more details, please see: paddle/fluid/memory/allocation/mmap_allocator.h # For more details, please see: paddle/fluid/memory/allocation/mmap_allocator.h
try: if os.environ.get('FLAGS_use_shm_cache', False) in [
self._worker_shm_buffer_size = (2 + 1) * len(self._dataset[0]) 1,
except: '1',
True,
'True',
'true',
]:
try:
self._worker_shm_buffer_size = (2 + 1) * len(self._dataset[0])
except:
self._worker_shm_buffer_size = 0
warnings.warn(
"Setting the shm cache buffer size to 0, equivalent to not using the shm cache policy."
)
else:
self._worker_shm_buffer_size = 0 self._worker_shm_buffer_size = 0
warnings.warn(
"Setting the shm cache buffer size to 0, equivalent to not using the shm cache policy."
)
self._main_thread_shm_buffer_size = ( self._main_thread_shm_buffer_size = (
(self._worker_shm_buffer_size) * 2 * self._num_workers (self._worker_shm_buffer_size) * 2 * self._num_workers
) )
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册