diff --git a/paddle/cuda/include/hl_matrix_type.cuh b/paddle/cuda/include/hl_matrix_type.cuh
index 2ced2fb1ab1afc8b0619a904709ce4504291dd86..77f73167fe6ea2e301b3f68e51d32f3e1578ec93 100644
--- a/paddle/cuda/include/hl_matrix_type.cuh
+++ b/paddle/cuda/include/hl_matrix_type.cuh
@@ -38,7 +38,7 @@ typedef double2 vecType;
 #endif
 #elif defined(__SSE3__)
 #include "hl_cpu_simd_sse.cuh"
-#elif defined(__ARM_NEON) || defined(__ARM_NEON__)
+#elif (defined(__ARM_NEON) || defined(__ARM_NEON__)) && !defined(__NVCC__)
 #include "hl_cpu_simd_neon.cuh"
 #else
 #include "hl_cpu_scalar.cuh"