diff --git a/paddle/fluid/pybind/tensor_py.h b/paddle/fluid/pybind/tensor_py.h
index d1d7681b7ba54775dd0064a12748a446ae700a20..9e5dc638516a958cc829eb034d0766e91771906d 100644
--- a/paddle/fluid/pybind/tensor_py.h
+++ b/paddle/fluid/pybind/tensor_py.h
@@ -187,21 +187,12 @@ void SetTensorFromPyArrayT(
     }
   } else {
 #ifdef PADDLE_WITH_CUDA
-    T *dst;
-    if (array.nbytes() <= 4 && !paddle::platform::is_cuda_pinned_place(place)) {
-      dst = self->mutable_data<T>(platform::CPUPlace());
-    } else {
-      dst = self->mutable_data<T>(place);
-    }
+    auto dst = self->mutable_data<T>(place);
     if (paddle::platform::is_cuda_pinned_place(place)) {
       std::memcpy(dst, array.data(), array.nbytes());
     } else if (paddle::platform::is_gpu_place(place)) {
-      if (array.nbytes() <= 4) {
-        std::memcpy(dst, array.data(), array.nbytes());
-      } else {
-        paddle::platform::GpuMemcpySync(dst, array.data(), array.nbytes(),
-                                        cudaMemcpyHostToDevice);
-      }
+      paddle::platform::GpuMemcpySync(dst, array.data(), array.nbytes(),
+                                      cudaMemcpyHostToDevice);
     } else {
       PADDLE_THROW(
           "Incompatible place type: Tensor.set() supports CPUPlace, CUDAPlace "