change the default memory copy way to async

mindspore/ccsrc/device/gpu/cuda_driver.cc

@@ -102,7 +102,7 @@ bool CudaDriver::CopyHostMemToDeviceAsync(const DeviceMemPtr &dst, const void *s
 
 bool CudaDriver::CopyDeviceMemToHostAsync(const HostMemPtr &dst, const DeviceMemPtr &src, size_t size,
                                           DeviceStream stream) {
-  auto ret = cudaMemcpyAsync(dst, src, size, cudaMemcpyHostToDevice, (cudaStream_t)stream);
+  auto ret = cudaMemcpyAsync(dst, src, size, cudaMemcpyDeviceToHost, (cudaStream_t)stream);
   if (ret != cudaSuccess) {
     MS_LOG(ERROR) << "cudaMemcpyAsync failed, ret[" << static_cast<int>(ret) << "], " << cudaGetErrorString(ret);
     return false;

mindspore/ccsrc/device/gpu/gpu_device_address.cc

@@ -37,7 +37,13 @@ bool GPUDeviceAddress::SyncDeviceToHost(const std::vector<int> &, size_t size, T
 
 bool GPUDeviceAddress::SyncHostToDevice(const std::vector<int> &, size_t, TypeId, const void *host_ptr) const {
   MS_EXCEPTION_IF_NULL(host_ptr);
-  return GPUDeviceManager::GetInstance().CopyHostMemToDevice(ptr_, host_ptr, size_);
+  auto &stream = GPUDeviceManager::GetInstance().default_stream();
+  MS_EXCEPTION_IF_NULL(stream);
+  if (!GPUDeviceManager::GetInstance().CopyHostMemToDeviceAsync(ptr_, host_ptr, size_, stream)) {
+    MS_LOG(ERROR) << "CopyHostMemToDeviceAsync failed";
+    return false;
+  }
+  return GPUDeviceManager::GetInstance().SyncStream(stream);
 }
 
 GPUDeviceAddress::~GPUDeviceAddress() {

mindspore/ccsrc/device/gpu/gpu_device_manager.cc

@@ -89,6 +89,16 @@ bool GPUDeviceManager::CopyDeviceMemToHost(const HostMemPtr &dst, const DeviceMe
 bool GPUDeviceManager::CopyHostMemToDevice(const DeviceMemPtr &dst, const void *src, size_t size) const {
   return CudaDriver::CopyHostMemToDevice(dst, src, size);
 }
+
+bool GPUDeviceManager::CopyDeviceMemToHostAsync(const HostMemPtr &dst, const DeviceMemPtr &src, size_t size,
+                                                DeviceStream stream) const {
+  return CudaDriver::CopyDeviceMemToHostAsync(dst, src, size, stream);
+}
+
+bool GPUDeviceManager::CopyHostMemToDeviceAsync(const DeviceMemPtr &dst, const void *src, size_t size,
+                                                DeviceStream stream) const {
+  return CudaDriver::CopyHostMemToDeviceAsync(dst, src, size, stream);
+}
 }  // namespace gpu
 }  // namespace device
 }  // namespace mindspore

mindspore/ccsrc/device/gpu/gpu_device_manager.h

@@ -47,6 +47,9 @@ class GPUDeviceManager {
   bool CopyDeviceMemToHost(const HostMemPtr &dst, const DeviceMemPtr &src, size_t size) const;
   bool CopyHostMemToDevice(const DeviceMemPtr &dst, const void *src, size_t size) const;
 
+  bool CopyDeviceMemToHostAsync(const HostMemPtr &dst, const DeviceMemPtr &src, size_t size, DeviceStream stream) const;
+  bool CopyHostMemToDeviceAsync(const DeviceMemPtr &dst, const void *src, size_t size, DeviceStream stream) const;
+
   static GPUDeviceManager &GetInstance() {
     static GPUDeviceManager instance;
     return instance;