diff --git a/paddle/fluid/operators/fused/fused_gemm_epilogue_op.h b/paddle/fluid/operators/fused/fused_gemm_epilogue_op.h
index 62701fe43f020a3f9d7610d80fe9dc869ae38c61..0f4fa9f88954db17400f8f59bed646fac2c3c30b 100644
--- a/paddle/fluid/operators/fused/fused_gemm_epilogue_op.h
+++ b/paddle/fluid/operators/fused/fused_gemm_epilogue_op.h
@@ -15,8 +15,12 @@ limitations under the License. */
 
 #pragma once
 
-#if defined(PADDLE_WITH_CUDA) && CUDA_VERSION >= 11060
+#ifdef PADDLE_WITH_CUDA
+
 #include <cuda_runtime_api.h>
+#include "cuda.h"  // NOLINT
+
+#if CUDA_VERSION >= 11060
 
 #include <algorithm>
 #include <mutex>
@@ -322,4 +326,6 @@ class GemmEpilogueAlgoCache {
 
 }  // namespace operators
 }  // namespace paddle
+
+#endif
 #endif