diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt
index a0d89394ece6a928679f06e485d258eb9d2aed5e..0cefb7e69ace31ce956c633c6e2697d3a125a199 100644
--- a/paddle/fluid/operators/CMakeLists.txt
+++ b/paddle/fluid/operators/CMakeLists.txt
@@ -175,12 +175,6 @@ endif()
 
 set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library")
 
-if(WITH_MKLDNN)
-include(mkldnn/inplace_op_tests.cmake)
-include(mkldnn/caching_tests.cmake)
-include(mkldnn/nhwc_op_tests.cmake)
-endif()
-
 if(WITH_UNITY_BUILD)
     # Using Unity Build to compile operators, `register_operator` will cause
     # the unity library to lose some symbols.
diff --git a/paddle/fluid/operators/mkldnn/caching_tests.cmake b/paddle/fluid/operators/mkldnn/caching_tests.cmake
deleted file mode 100644
index 49f08622265d00bddb3a73dec3c10a4b28882c06..0000000000000000000000000000000000000000
--- a/paddle/fluid/operators/mkldnn/caching_tests.cmake
+++ /dev/null
@@ -1,20 +0,0 @@
-set(TEST_MKLDNN_CACHING_DEPS
-    op_registry
-    elementwise_mul_op
-    elementwise_add_op
-    activation_op
-    softmax_op
-    conv_op
-    im2col
-    vol2col
-    softmax
-    scope
-    device_context
-    enforce)
-if(WITH_GPU OR WITH_ROCM)
-  set(TEST_MKLDNN_CACHING_DEPS ${TEST_MKLDNN_CACHING_DEPS} depthwise_conv)
-endif()
-cc_test(
-  test_mkldnn_caching
-  SRCS mkldnn/test_mkldnn_caching.cc
-  DEPS ${TEST_MKLDNN_CACHING_DEPS})
diff --git a/paddle/fluid/operators/mkldnn/inplace_op_tests.cmake b/paddle/fluid/operators/mkldnn/inplace_op_tests.cmake
deleted file mode 100644
index 18893e22ec85be5dd71b962c5b39f703155249c2..0000000000000000000000000000000000000000
--- a/paddle/fluid/operators/mkldnn/inplace_op_tests.cmake
+++ /dev/null
@@ -1,12 +0,0 @@
-cc_test(
-  test_mkldnn_op_inplace
-  SRCS mkldnn/test_mkldnn_op_inplace.cc
-  DEPS op_registry
-       elementwise_add_op
-       activation_op
-       softmax_op
-       softmax
-       scope
-       device_context
-       enforce
-       executor)
diff --git a/paddle/fluid/operators/mkldnn/nhwc_op_tests.cmake b/paddle/fluid/operators/mkldnn/nhwc_op_tests.cmake
deleted file mode 100644
index bda7ab2a3b3a24188f8270ed9c72152b7824311e..0000000000000000000000000000000000000000
--- a/paddle/fluid/operators/mkldnn/nhwc_op_tests.cmake
+++ /dev/null
@@ -1,18 +0,0 @@
-cc_test_old(
-  test_mkldnn_op_nhwc
-  SRCS
-  mkldnn/test_mkldnn_op_nhwc.cc
-  DEPS
-  op_registry
-  pool_op
-  shape_op
-  crop_op
-  activation_op
-  generated_op
-  pooling
-  transpose_op
-  fused_transpose_op
-  scope
-  device_context
-  enforce
-  executor)
diff --git a/test/cpp/fluid/CMakeLists.txt b/test/cpp/fluid/CMakeLists.txt
index 029cf3db9053db8199df2275712ef4920f1871ff..34f45371cca2ab0b499fb8b48d28a65cdbab7256 100644
--- a/test/cpp/fluid/CMakeLists.txt
+++ b/test/cpp/fluid/CMakeLists.txt
@@ -11,9 +11,9 @@ if(WITH_LITE)
   add_subdirectory(lite)
 endif()
 add_subdirectory(math)
-# if(WITH_MKLDNN)
-#   add_subdirectory(mkldnn)
-# endif()
+if(WITH_MKLDNN)
+  add_subdirectory(mkldnn)
+endif()
 add_subdirectory(nccl)
 if(WITH_PSCORE)
   add_subdirectory(pscore)
diff --git a/test/cpp/fluid/mkldnn/CMakeLists.txt b/test/cpp/fluid/mkldnn/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..34f3ce8959c6839d7fdd18c02ff1400f987f4908
--- /dev/null
+++ b/test/cpp/fluid/mkldnn/CMakeLists.txt
@@ -0,0 +1,52 @@
+cc_test(
+  test_mkldnn_op_inplace
+  SRCS test_mkldnn_op_inplace.cc
+  DEPS op_registry
+       elementwise_add_op
+       activation_op
+       softmax_op
+       softmax
+       scope
+       device_context
+       enforce
+       executor)
+
+set(TEST_MKLDNN_CACHING_DEPS
+    op_registry
+    elementwise_mul_op
+    elementwise_add_op
+    activation_op
+    softmax_op
+    conv_op
+    im2col
+    vol2col
+    softmax
+    scope
+    device_context
+    enforce)
+if(WITH_GPU OR WITH_ROCM)
+  set(TEST_MKLDNN_CACHING_DEPS ${TEST_MKLDNN_CACHING_DEPS} depthwise_conv)
+endif()
+cc_test(
+  test_mkldnn_caching
+  SRCS test_mkldnn_caching.cc
+  DEPS ${TEST_MKLDNN_CACHING_DEPS})
+
+cc_test_old(
+  test_mkldnn_op_nhwc
+  SRCS
+  test_mkldnn_op_nhwc.cc
+  DEPS
+  op_registry
+  pool_op
+  shape_op
+  crop_op
+  activation_op
+  generated_op
+  pooling
+  transpose_op
+  fused_transpose_op
+  scope
+  device_context
+  enforce
+  executor)
diff --git a/paddle/fluid/operators/mkldnn/test_mkldnn_caching.cc b/test/cpp/fluid/mkldnn/test_mkldnn_caching.cc
similarity index 100%
rename from paddle/fluid/operators/mkldnn/test_mkldnn_caching.cc
rename to test/cpp/fluid/mkldnn/test_mkldnn_caching.cc
diff --git a/paddle/fluid/operators/mkldnn/test_mkldnn_op_inplace.cc b/test/cpp/fluid/mkldnn/test_mkldnn_op_inplace.cc
similarity index 100%
rename from paddle/fluid/operators/mkldnn/test_mkldnn_op_inplace.cc
rename to test/cpp/fluid/mkldnn/test_mkldnn_op_inplace.cc
diff --git a/paddle/fluid/operators/mkldnn/test_mkldnn_op_nhwc.cc b/test/cpp/fluid/mkldnn/test_mkldnn_op_nhwc.cc
similarity index 100%
rename from paddle/fluid/operators/mkldnn/test_mkldnn_op_nhwc.cc
rename to test/cpp/fluid/mkldnn/test_mkldnn_op_nhwc.cc