diff --git a/cmake/configure.cmake b/cmake/configure.cmake
index 51c3b918cc4ef4cf6c8052ccc14028a872309fcf..4e044ca4216a7c1bd2668358c544d4f970435083 100644
--- a/cmake/configure.cmake
+++ b/cmake/configure.cmake
@@ -49,11 +49,16 @@ if(NOT WITH_GOLANG)
 endif(NOT WITH_GOLANG)
 
 if(NOT WITH_GPU)
+    # Will gradually remove uses of PADDLE_ONLY_CPU in source files,
+    # so could we remove -DPADDLE_ONLY_CPU.
+    # c.f. https://github.com/PaddlePaddle/Paddle/issues/4588
     add_definitions(-DPADDLE_ONLY_CPU)
     add_definitions(-DHPPL_STUB_FUNC)
 
     list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu)
 else()
+    add_definitions(-DPADDLE_WITH_CUDA)
+
     FIND_PACKAGE(CUDA REQUIRED)
 
     if(${CUDA_VERSION_MAJOR} VERSION_LESS 7)