diff --git a/paddle/fluid/operators/math/CMakeLists.txt b/paddle/fluid/operators/math/CMakeLists.txt
index e174c8992c26c5c849fa5ef9261a139547efd5ad..77a8aa5d9c7a1691d9c667c26e0bbbba688bb4ef 100644
--- a/paddle/fluid/operators/math/CMakeLists.txt
+++ b/paddle/fluid/operators/math/CMakeLists.txt
@@ -8,6 +8,9 @@ function(math_library TARGET)
     set(cu_srcs)
     set(hip_srcs)
     set(math_common_deps device_context framework_proto enforce)
+    if (WITH_GPU)
+        list(APPEND math_common_deps cub)
+    endif()
     set(multiValueArgs DEPS)
     cmake_parse_arguments(math_library "${options}" "${oneValueArgs}"
             "${multiValueArgs}" ${ARGN})
@@ -37,7 +40,7 @@ math_library(concat_and_split)
 math_library(context_project DEPS im2col math_function)
 math_library(cross_entropy)
 math_library(cos_sim_functor)
-math_library(depthwise_conv DEPS cub)
+math_library(depthwise_conv)
 math_library(im2col)
 math_library(sample_prob)
 math_library(sampler)