diff --git a/paddle/pten/CMakeLists.txt b/paddle/pten/CMakeLists.txt
index 7adfca40319b1ba5c415a5122132d2e7aa504b38..97be4c19c970b4e08416108096e4f9a093f08796 100644
--- a/paddle/pten/CMakeLists.txt
+++ b/paddle/pten/CMakeLists.txt
@@ -23,13 +23,12 @@ add_subdirectory(ops)
 add_subdirectory(tests)
 
 # make an unity target for compile deps
-set(PTEN_DEPS convert_utils dense_tensor pten_context kernel_factory kernel_context)
+set(PTEN_DEPS convert_utils dense_tensor pten_context kernel_factory kernel_context infermeta)
 get_property(pten_kernels GLOBAL PROPERTY PTEN_KERNELS)
 # keep this message for debug, remove it later if needless
 message(STATUS "All standard pten kernels: ${pten_kernels}")
 set(PTEN_DEPS ${PTEN_DEPS} ${pten_kernels})
 set(PTEN_DEPS ${PTEN_DEPS} math_cpu)
-set(PTEN_DEPS ${PTEN_DEPS} nary unary binary)
 if(WITH_GPU OR WITH_ROCM)
   set(PTEN_DEPS ${PTEN_DEPS} math_gpu)
 endif()
diff --git a/paddle/pten/infermeta/CMakeLists.txt b/paddle/pten/infermeta/CMakeLists.txt
index b32ec0a51c7367838adf19b42c55a577e599f966..2b4bba8313f58d5abb3fd4fa385b9c3596c11fb8 100644
--- a/paddle/pten/infermeta/CMakeLists.txt
+++ b/paddle/pten/infermeta/CMakeLists.txt
@@ -1,3 +1 @@
-cc_library(nary SRCS nary.cc DEPS convert_utils)
-cc_library(unary SRCS unary.cc DEPS convert_utils)
-cc_library(binary SRCS binary.cc DEPS convert_utils)
+cc_library(infermeta SRCS nary.cc unary.cc binary.cc DEPS convert_utils)
diff --git a/paddle/pten/kernels/CMakeLists.txt b/paddle/pten/kernels/CMakeLists.txt
index be4f0d5b191ff326a5e8aab416196a8608f4236e..7a785d86921aa3ca48b15b6824f7d9066c0a9aff 100644
--- a/paddle/pten/kernels/CMakeLists.txt
+++ b/paddle/pten/kernels/CMakeLists.txt
@@ -26,6 +26,7 @@ set_property(GLOBAL PROPERTY PTEN_KERNELS "")
 
 set(COMMON_KERNEL_DEPS dense_tensor kernel_context kernel_factory)
 set(COMMON_KERNEL_DEPS ${COMMON_KERNEL_DEPS} eigen_function)
+set(COMMON_KERNEL_DEPS ${COMMON_KERNEL_DEPS} infermeta)
 
 # auto build kernel targets by cmake
 register_kernels(DEPS ${COMMON_KERNEL_DEPS})
diff --git a/paddle/pten/kernels/matmul_kernel.h b/paddle/pten/kernels/matmul_kernel.h
index 5a1766330b7781a169d8082eaf41ab317e9ff888..fb54a5301e61c0d3e89a860061372cb1dbf55a42 100644
--- a/paddle/pten/kernels/matmul_kernel.h
+++ b/paddle/pten/kernels/matmul_kernel.h
@@ -16,7 +16,7 @@
 
 #include "paddle/pten/api/lib/utils/storage.h"
 #include "paddle/pten/core/dense_tensor.h"
-#include "paddle/pten/include/infermeta.h"
+#include "paddle/pten/infermeta/binary.h"
 
 namespace pten {