From 770501b802e9371059fe0b308c7c0c3f1ab70d75 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Thu, 13 Oct 2022 19:16:25 +0800 Subject: [PATCH] add thread name for dataloader (#46990) --- paddle/fluid/platform/os_info.cc | 2 ++ paddle/fluid/platform/os_info.h | 2 +- paddle/fluid/pybind/pybind.cc | 2 ++ python/paddle/fluid/dataloader/dataloader_iter.py | 2 ++ python/paddle/fluid/layers/io.py | 1 + python/paddle/fluid/reader.py | 3 +++ 6 files changed, 11 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/platform/os_info.cc b/paddle/fluid/platform/os_info.cc index 694f701b5a..ec85619966 100644 --- a/paddle/fluid/platform/os_info.cc +++ b/paddle/fluid/platform/os_info.cc @@ -27,6 +27,7 @@ limitations under the License. */ #else #include #endif +#include "glog/logging.h" #include "paddle/fluid/framework/new_executor/workqueue/thread_data_registry.h" #include "paddle/fluid/platform/macros.h" // import DISABLE_COPY_AND_ASSIGN @@ -115,6 +116,7 @@ bool SetCurrentThreadName(const std::string& name) { return false; } instance.SetCurrentThreadData(name); + VLOG(4) << __func__ << " " << name; return true; } diff --git a/paddle/fluid/platform/os_info.h b/paddle/fluid/platform/os_info.h index ef894fd3dc..a827d063c1 100644 --- a/paddle/fluid/platform/os_info.h +++ b/paddle/fluid/platform/os_info.h @@ -57,7 +57,7 @@ ThreadId GetCurrentThreadId(); // create/destory when using it. std::unordered_map GetAllThreadIds(); -static constexpr const char* kDefaultThreadName = "unset"; +static constexpr const char* kDefaultThreadName = "unnamed"; // Returns kDefaultThreadName if SetCurrentThreadName is never called. std::string GetCurrentThreadName(); diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 410f2d19de..71f1cf8184 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -854,6 +854,8 @@ PYBIND11_MODULE(libpaddle, m) { m.def("_set_paddle_lib_path", &paddle::platform::dynload::SetPaddleLibPath); + m.def("set_current_thread_name", &paddle::platform::SetCurrentThreadName); + m.def("_promote_types_if_complex_exists", &paddle::framework::PromoteTypesIfComplexExists); diff --git a/python/paddle/fluid/dataloader/dataloader_iter.py b/python/paddle/fluid/dataloader/dataloader_iter.py index 92fe3fb915..71e4a9f83a 100644 --- a/python/paddle/fluid/dataloader/dataloader_iter.py +++ b/python/paddle/fluid/dataloader/dataloader_iter.py @@ -205,6 +205,7 @@ class _DataLoaderIterSingleProcess(_DataLoaderIterBase): # If we do not set cudaDeviceId in new thread, the default cudaDeviceId will be 0, # Which may cost hundreds of MB of GPU memory on CUDAPlace(0) if calling some cuda # APIs in this thread. + core.set_current_thread_name("Dataloader_" + str(id(self))) _set_expected_place(legacy_expected_place) while not self._thread_done_event.is_set(): @@ -530,6 +531,7 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): # If we do not set cudaDeviceId in new thread, the default cudaDeviceId will be 0, # Which may cost hundreds of MB of GPU memory on CUDAPlace(0) if calling some cuda # APIs in this thread. + core.set_current_thread_name("Dataloader_" + str(id(self))) _set_expected_place(legacy_expected_place) while not self._thread_done_event.is_set(): diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 62b31cdbed..c526bd655d 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -477,6 +477,7 @@ def _py_reader(capacity, def __provider_thread__(legacy_expected_place): try: # See _DataLoaderIterSingleProcess._thread_loop() for why set expected place here. + _set_expected_place(legacy_expected_place) for tensors in func(): diff --git a/python/paddle/fluid/reader.py b/python/paddle/fluid/reader.py index 0eed36fb12..07dd03a2bc 100644 --- a/python/paddle/fluid/reader.py +++ b/python/paddle/fluid/reader.py @@ -1126,6 +1126,7 @@ class DygraphGeneratorLoader(DataLoaderBase): def _reader_thread_loop_for_multiprocess(self, legacy_expected_place): # See _DataLoaderIterSingleProcess._thread_loop() for why set expected place here. + core.set_current_thread_name("Dataloader_" + str(id(self))) _set_expected_place(legacy_expected_place) while not self._thread_done_event.is_set(): @@ -1169,6 +1170,7 @@ class DygraphGeneratorLoader(DataLoaderBase): def _reader_thread_loop_for_singleprocess(self, legacy_expected_place): try: # See _DataLoaderIterSingleProcess._thread_loop() for why set expected place here. + core.set_current_thread_name("Dataloader_" + str(id(self))) _set_expected_place(legacy_expected_place) for sample in self._batch_reader(): @@ -1419,6 +1421,7 @@ class GeneratorLoader(DataLoaderBase): def __thread_main__(legacy_expected_place): try: # See _DataLoaderIterSingleProcess._thread_loop() for why set expected place here. + core.set_current_thread_name("Dataloader_" + str(id(self))) _set_expected_place(legacy_expected_place) while not self._queue.wait_for_inited(1): -- GitLab