From a91b80146f67b22385d9548d7e35da3f42ed1450 Mon Sep 17 00:00:00 2001
From: liuwei1031 <46661762+liuwei1031@users.noreply.github.com>
Date: Mon, 25 Nov 2019 11:11:14 +0800
Subject: [PATCH] cherry-pick (#21201) to release/1.6 (#21306)

cudaStreamSynchronize randomly hang when used in multi-thread environment, replace it with cudaStreamQuery API on windows
---
 paddle/fluid/platform/device_context.cc | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/paddle/fluid/platform/device_context.cc b/paddle/fluid/platform/device_context.cc
index 31665933654..b19929127ee 100644
--- a/paddle/fluid/platform/device_context.cc
+++ b/paddle/fluid/platform/device_context.cc
@@ -313,14 +313,23 @@ CUDADeviceContext::~CUDADeviceContext() {
 Place CUDADeviceContext::GetPlace() const { return place_; }
 
 void CUDADeviceContext::Wait() const {
-  cudaError_t e_sync = cudaStreamSynchronize(stream_);
-  if (e_sync != 0) {
+  cudaError_t e_sync = cudaSuccess;
+#if !defined(_WIN32)
+  e_sync = cudaStreamSynchronize(stream_);
+#else
+  while (e_sync = cudaStreamQuery(stream_)) {
+    if (e_sync == cudaErrorNotReady) continue;
+    break;
+  }
+#endif
+
+  if (cudaSuccess != e_sync) {
     LOG(FATAL) << "cudaStreamSynchronize " << cudaGetErrorString(e_sync)
                << " errno: " << e_sync;
   }
 
   cudaError_t e_get = cudaGetLastError();
-  if (e_get != 0) {
+  if (cudaSuccess != e_get) {
     LOG(FATAL) << "cudaGetLastError  " << cudaGetErrorString(e_get)
                << " errno: " << e_get;
   }
-- 
GitLab