diff --git a/paddle/fluid/operators/math/CMakeLists.txt b/paddle/fluid/operators/math/CMakeLists.txt
index 5ae4a00d3f3fb81608d4a4d9dc1162c0efecaabc..6f672a069bc2521945c45e706d961f3ef40ef3c6 100644
--- a/paddle/fluid/operators/math/CMakeLists.txt
+++ b/paddle/fluid/operators/math/CMakeLists.txt
@@ -41,7 +41,11 @@ math_library(depthwise_conv)
 math_library(gru_compute DEPS activation_functions math_function)
 math_library(im2col)
 math_library(lstm_compute DEPS activation_functions)
-cc_library(blas SRCS blas.cc DEPS cblas libxsmm framework_proto device_context)
+set(BLAS_DEPS cblas framework_proto device_context)
+if (WITH_LIBXSMM)
+  list(APPEND BLAS_DEPS libxsmm)
+endif()
+cc_library(blas SRCS blas.cc DEPS ${BLAS_DEPS})
 math_library(math_function DEPS blas)
 math_library(maxouting)
 math_library(pooling)