diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt
index 3cbbbcb3284912ca9e6da92412e182bd279da541..5d436a7e0c3752c889c19820507589f34d3bee94 100644
--- a/paddle/fluid/operators/CMakeLists.txt
+++ b/paddle/fluid/operators/CMakeLists.txt
@@ -201,7 +201,7 @@ op_library(save_op DEPS lod_tensor)
 op_library(load_op DEPS lod_tensor)
 op_library(save_combine_op DEPS lod_tensor)
 op_library(load_combine_op DEPS lod_tensor)
-op_library(concat_op DEPS concat_functor)
+op_library(concat_op DEPS concat)
 
 list(REMOVE_ITEM GENERAL_OPS ${DEPS_OPS})
 foreach(src ${GENERAL_OPS})
diff --git a/paddle/fluid/operators/math/CMakeLists.txt b/paddle/fluid/operators/math/CMakeLists.txt
index e88f4ed1dc13d0f8344e6ac56ae0d575c3b02ac3..11bc176400398cd99f244f2ba9015b8c3c0093fb 100644
--- a/paddle/fluid/operators/math/CMakeLists.txt
+++ b/paddle/fluid/operators/math/CMakeLists.txt
@@ -6,8 +6,8 @@ function(math_library TARGET)
     # But it handle split GPU/CPU code and link some common library.
     set(cc_srcs)
     set(cu_srcs)
-    set(math_common_deps device_context framework_proto)
-    set(multiValueArgs SRCS DEPS)
+    set(math_common_deps device_context)
+    set(multiValueArgs DEPS)
     cmake_parse_arguments(math_library "${options}" "${oneValueArgs}"
             "${multiValueArgs}" ${ARGN})
 
@@ -18,36 +18,34 @@ function(math_library TARGET)
         list(APPEND cu_srcs ${TARGET}.cu)
     endif()
 
+    list(LENGTH cc_srcs cc_srcs_len)
     if (WITH_GPU)
         nv_library(${TARGET} SRCS ${cc_srcs} ${cu_srcs} DEPS ${math_library_DEPS} ${math_common_deps})
-    else()
+    elseif(${cc_srcs_len} GREATER 0)
         cc_library(${TARGET} SRCS ${cc_srcs} DEPS ${math_library_DEPS} ${math_common_deps})
     endif()
 endfunction()
 
-math_library(math_function DEPS cblas)
-math_library(im2col)
-math_library(selected_rows_functor DEPS selected_rows)
-math_library(softmax)
+# please add new math_library in alphabetical order
+math_library(concat)
+math_library(context_project DEPS im2col math_function)
 math_library(cross_entropy)
+math_library(cos_sim_functor)
+math_library(depthwise_conv)
+math_library(gru_compute DEPS activation_functions math_function)
+math_library(im2col)
+math_library(lstm_compute DEPS activation_functions)
+math_library(math_function DEPS cblas framework_proto)
+math_library(maxouting)
 math_library(pooling)
-math_library(sequence_pooling)
-math_library(vol2col)
-math_library(context_project)
+math_library(selected_rows_functor DEPS selected_rows)
 math_library(sequence2batch)
 math_library(sequence_padding)
+math_library(sequence_pooling DEPS math_function)
 math_library(sequence_scale)
-math_library(maxouting)
+math_library(softmax)
 math_library(unpooling)
-math_library(cos_sim_functor)
-math_library(lstm_compute DEPS activation_functions)
-math_library(gru_compute DEPS activation_functions)
-if(WITH_GPU)
-    nv_library(depthwise_conv SRCS depthwise_conv.cu DEPS device_context)
-    nv_library(concat_functor SRCS concat.cc concat.cu DEPS device_context tensor)
-else()
-    cc_library(concat_functor SRCS concat.cc DEPS device_context tensor)
-endif()
+math_library(vol2col)
 
 cc_test(math_function_test SRCS math_function_test.cc)
 cc_test(selected_rows_functor_test SRCS selected_rows_functor_test.cc DEPS selected_rows_functor)
@@ -58,4 +56,4 @@ if(WITH_GPU)
     nv_test(math_function_gpu_test SRCS math_function_test.cu)
     nv_test(selected_rows_functor_gpu_test SRCS selected_rows_functor_test.cu DEPS selected_rows_functor)
 endif()
-cc_test(concat_test SRCS concat_test.cc DEPS concat_functor tensor)
+cc_test(concat_test SRCS concat_test.cc DEPS concat)
diff --git a/paddle/fluid/operators/math/sequence2batch.cc b/paddle/fluid/operators/math/sequence2batch.cc
index 72bf2ab17016157f22bcf180090660346ed4f60d..8899abff360ea867872d3433722cdb37ef358500 100644
--- a/paddle/fluid/operators/math/sequence2batch.cc
+++ b/paddle/fluid/operators/math/sequence2batch.cc
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "paddle/fluid/operators/math/sequence2batch.h"
-#include "paddle/fluid/operators/math/math_function.h"
 
 namespace paddle {
 namespace operators {