From 6553a9d7a355c4e9ef04a0cd42702b1d36b46700 Mon Sep 17 00:00:00 2001 From: Ruibiao Chen Date: Mon, 25 Apr 2022 13:25:02 +0800 Subject: [PATCH] Do not reset default stream for StreamSafeCUDAAllocator (#42149) --- .../fluid/memory/allocation/allocator_facade.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/paddle/fluid/memory/allocation/allocator_facade.cc b/paddle/fluid/memory/allocation/allocator_facade.cc index e2730a1b825..e2649a7fd33 100644 --- a/paddle/fluid/memory/allocation/allocator_facade.cc +++ b/paddle/fluid/memory/allocation/allocator_facade.cc @@ -415,6 +415,23 @@ class AllocatorFacadePrivate { void SetDefaultStream(const platform::CUDAPlace& place, gpuStream_t stream) { const std::shared_ptr& allocator = GetDefaultStreamSafeCUDAAllocator(place); + + // NOTE(Ruibiao): The default stream will be set when the CUDADeviceContext + // created. Normally, the DeviceContextPool is a global singleton and one + // Place only correspond to one DeviceContext. However, to support + // multi-stream scheduling, standalone executor creates two extra + // DeviceContextPools for H2D and D2H stream in StreamAnalyzer, which make + // one Place correspond to multiple DeviceContext and unexpectedly reset the + // default stream in runtime. To avoid this behavior, we do not allow + // changing default stream after initially setting. + if (allocator->GetDefaultStream() != nullptr) { + VLOG(5) << "The default stream for StreamSafeCUDAAllocator(" + << allocator.get() << ") in " << place << " has been set to " + << allocator->GetDefaultStream() + << " before, not allow to change now."; + return; + } + allocator->SetDefaultStream(stream); VLOG(8) << "Set default stream to " << stream << " for StreamSafeCUDAAllocator(" << allocator.get() << ") in " -- GitLab