diff --git a/paddle/pten/kernels/cpu/creation.h b/paddle/pten/kernels/cpu/creation.h
index 9d53f78026a52803de84ce5ef5bdf492c48c35d3..75d99c0794c40a748116612175161acd4ac699b0 100644
--- a/paddle/pten/kernels/cpu/creation.h
+++ b/paddle/pten/kernels/cpu/creation.h
@@ -14,16 +14,13 @@
 
 #pragma once
 
+#include "paddle/pten/backends/cpu/cpu_context.h"
 #include "paddle/pten/common/scalar.h"
 #include "paddle/pten/common/scalar_array.h"
 #include "paddle/pten/core/dense_tensor.h"
 
-#include "paddle/fluid/platform/device_context.h"
-
 namespace pten {
 
-using CPUContext = paddle::platform::CPUDeviceContext;
-
 template <typename T>
 void FullLike(const CPUContext& dev_ctx, const Scalar& val, DenseTensor* out);
 
diff --git a/paddle/pten/kernels/cpu/linalg.h b/paddle/pten/kernels/cpu/linalg.h
index a954033866f177cc37db15a78668e6b0c1fe1cf7..29c6cd16cf81a7c12850a31ba46fe4508b881c90 100644
--- a/paddle/pten/kernels/cpu/linalg.h
+++ b/paddle/pten/kernels/cpu/linalg.h
@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include "paddle/pten/backends/cpu/cpu_context.h"
 #include "paddle/pten/core/dense_tensor.h"
 
 // See Note [ Why still include the fluid headers? ]
@@ -21,8 +22,6 @@
 
 namespace pten {
 
-using CPUContext = paddle::platform::CPUDeviceContext;
-
 template <typename T>
 void Dot(const CPUContext& dev_ctx,
          const DenseTensor& x,
diff --git a/paddle/pten/kernels/cpu/manipulation.h b/paddle/pten/kernels/cpu/manipulation.h
index cc58354787585e9283e6d9000bbc1114980e31c8..1a219dc79e601aa971f7135906b8e0cdb4bde103 100644
--- a/paddle/pten/kernels/cpu/manipulation.h
+++ b/paddle/pten/kernels/cpu/manipulation.h
@@ -14,17 +14,13 @@ limitations under the License. */
 
 #pragma once
 
+#include "paddle/pten/backends/cpu/cpu_context.h"
 #include "paddle/pten/common/scalar_array.h"
 #include "paddle/pten/core/dense_tensor.h"
 #include "paddle/pten/core/kernel_registry.h"
 
-// See Note [ Why still include the fluid headers? ]
-#include "paddle/fluid/platform/device_context.h"
-
 namespace pten {
 
-using CPUContext = paddle::platform::CPUDeviceContext;
-
 template <typename T>
 void Flatten(const CPUContext& dev_ctx,
              const DenseTensor& x,
diff --git a/paddle/pten/kernels/cpu/math.h b/paddle/pten/kernels/cpu/math.h
index c53e659cf83200c48b750b8085d98795e820ed2f..67a2feb4eef837b6aa89819e4e9d3b5c669d928c 100644
--- a/paddle/pten/kernels/cpu/math.h
+++ b/paddle/pten/kernels/cpu/math.h
@@ -14,17 +14,13 @@ limitations under the License. */
 
 #pragma once
 
+#include "paddle/pten/backends/cpu/cpu_context.h"
 #include "paddle/pten/common/scalar.h"
 #include "paddle/pten/core/dense_tensor.h"
 #include "paddle/pten/core/kernel_registry.h"
 
-// See Note [ Why still include the fluid headers? ]
-#include "paddle/fluid/platform/device_context.h"
-
 namespace pten {
 
-using CPUContext = paddle::platform::CPUDeviceContext;
-
 template <typename T>
 void Sign(const CPUContext& dev_ctx, const DenseTensor& x, DenseTensor* out);
 
diff --git a/paddle/pten/kernels/cpu/utils.h b/paddle/pten/kernels/cpu/utils.h
index 527346f005f900ab46c5da4122180869062d9e2a..93730692079e3e6ab6c6fac9578c9b108d66c63f 100644
--- a/paddle/pten/kernels/cpu/utils.h
+++ b/paddle/pten/kernels/cpu/utils.h
@@ -14,15 +14,12 @@ limitations under the License. */
 
 #pragma once
 
+#include "paddle/pten/backends/cpu/cpu_context.h"
 #include "paddle/pten/core/dense_tensor.h"
 #include "paddle/pten/core/kernel_registry.h"
 
-// See Note [ Why still include the fluid headers? ]
-#include "paddle/fluid/platform/device_context.h"
 namespace pten {
 
-using CPUContext = paddle::platform::CPUDeviceContext;
-
 void Copy(const CPUContext& dev_ctx,
           const DenseTensor& src,
           bool blocking,
diff --git a/paddle/pten/kernels/cuda/creation.h b/paddle/pten/kernels/cuda/creation.h
index a8fa7dc84d45a98b1b20d501f5e107f58d85d73b..72e8fbd017490a1629b1784ebf506a93c29a14ad 100644
--- a/paddle/pten/kernels/cuda/creation.h
+++ b/paddle/pten/kernels/cuda/creation.h
@@ -17,16 +17,13 @@
 // CUDA and HIP use same api
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 
+#include "paddle/pten/backends/cuda/cuda_context.h"
 #include "paddle/pten/common/scalar.h"
 #include "paddle/pten/common/scalar_array.h"
 #include "paddle/pten/core/dense_tensor.h"
 
-#include "paddle/fluid/platform/device_context.h"
-
 namespace pten {
 
-using CUDAContext = paddle::platform::CUDADeviceContext;
-
 template <typename T>
 void FullLike(const CUDAContext& dev_ctx, const Scalar& val, DenseTensor* out);
 
diff --git a/paddle/pten/kernels/cuda/linalg.h b/paddle/pten/kernels/cuda/linalg.h
index a6489efa72eee2a6bb3384da019249afea5b03fb..84f48ca609b270f999e4a8479ab8ea8f34c9773d 100644
--- a/paddle/pten/kernels/cuda/linalg.h
+++ b/paddle/pten/kernels/cuda/linalg.h
@@ -17,15 +17,11 @@
 // CUDA and HIP use same api
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 
+#include "paddle/pten/backends/cuda/cuda_context.h"
 #include "paddle/pten/core/dense_tensor.h"
 
-// See Note [ Why still include the fluid headers? ]
-#include "paddle/fluid/platform/device_context.h"
-
 namespace pten {
 
-using CUDAContext = paddle::platform::CUDADeviceContext;
-
 template <typename T>
 void Dot(const CUDAContext& dev_ctx,
          const DenseTensor& x,
diff --git a/paddle/pten/kernels/cuda/manipulation.h b/paddle/pten/kernels/cuda/manipulation.h
index be935a045f9384350aea2cef4fe929ef0932f775..165b08ad59a1cf4e123686dcb8d4e73178a6974b 100644
--- a/paddle/pten/kernels/cuda/manipulation.h
+++ b/paddle/pten/kernels/cuda/manipulation.h
@@ -17,17 +17,13 @@
 // CUDA and HIP use same api
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 
+#include "paddle/pten/backends/cuda/cuda_context.h"
 #include "paddle/pten/common/scalar_array.h"
 #include "paddle/pten/core/dense_tensor.h"
 #include "paddle/pten/core/kernel_registry.h"
 
-// See Note [ Why still include the fluid headers? ]
-#include "paddle/fluid/platform/device_context.h"
-
 namespace pten {
 
-using CUDAContext = paddle::platform::CUDADeviceContext;
-
 template <typename T>
 void Flatten(const CUDAContext& dev_ctx,
              const DenseTensor& x,
diff --git a/paddle/pten/kernels/cuda/math.h b/paddle/pten/kernels/cuda/math.h
index 37f61c51249d9b5e36cbe50af198b11a95f62c1e..c9f2f04494f84f24ff62d537b550a3d32f87922a 100644
--- a/paddle/pten/kernels/cuda/math.h
+++ b/paddle/pten/kernels/cuda/math.h
@@ -17,16 +17,12 @@ limitations under the License. */
 // CUDA and HIP use same api
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 
+#include "paddle/pten/backends/cuda/cuda_context.h"
 #include "paddle/pten/common/scalar.h"
 #include "paddle/pten/core/dense_tensor.h"
 
-// See Note [ Why still include the fluid headers? ]
-#include "paddle/fluid/platform/device_context.h"
-
 namespace pten {
 
-using CUDAContext = paddle::platform::CUDADeviceContext;
-
 template <typename T>
 void Sign(const CUDAContext& dev_ctx, const DenseTensor& x, DenseTensor* out);
 
diff --git a/paddle/pten/kernels/cuda/utils.h b/paddle/pten/kernels/cuda/utils.h
index d375c1cea23812956d27c330b9b0400dd4de5382..bd29347e3136513535a385222c0bae124a8953ee 100644
--- a/paddle/pten/kernels/cuda/utils.h
+++ b/paddle/pten/kernels/cuda/utils.h
@@ -17,15 +17,12 @@ limitations under the License. */
 // CUDA and HIP use same api
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 
+#include "paddle/pten/backends/cuda/cuda_context.h"
 #include "paddle/pten/core/dense_tensor.h"
 #include "paddle/pten/core/kernel_registry.h"
 
-// See Note [ Why still include the fluid headers? ]
-#include "paddle/fluid/platform/device_context.h"
 namespace pten {
 
-using CUDAContext = paddle::platform::CUDADeviceContext;
-
 void Copy(const CUDAContext& dev_ctx,
           const DenseTensor& src,
           bool blocking,
diff --git a/paddle/pten/kernels/hybird/CMakeLists.txt b/paddle/pten/kernels/hybird/CMakeLists.txt
index c82cbd1ef9e21a6b71c1e8d2bda6c105391357d6..9d4d86f0674a962edd5a8300fdfb1152716c35df 100644
--- a/paddle/pten/kernels/hybird/CMakeLists.txt
+++ b/paddle/pten/kernels/hybird/CMakeLists.txt
@@ -2,9 +2,9 @@ add_subdirectory(eigen)
 add_subdirectory(blas)
 add_subdirectory(general)
 
-cc_library(pten_transpose_cpu SRCS transpose.cc DEPS dense_tensor device_context)
+cc_library(pten_transpose_cpu SRCS transpose.cc DEPS dense_tensor pten_context)
 if(WITH_GPU)
-  nv_library(pten_transpose_cuda SRCS transpose.cu DEPS dense_tensor malloc device_context)
+  nv_library(pten_transpose_cuda SRCS transpose.cu DEPS dense_tensor malloc pten_context)
 elseif(WITH_ROCM)
-  hip_library(pten_transpose_cuda SRCS transpose.cu DEPS dense_tensor malloc device_context)
+  hip_library(pten_transpose_cuda SRCS transpose.cu DEPS dense_tensor malloc pten_context)
 endif()
diff --git a/paddle/pten/kernels/hybird/cuda/cast_kernel_impl.h b/paddle/pten/kernels/hybird/cuda/cast_kernel_impl.h
index 1bf5bb288e83200a7e2618d86ff1ae4dd9856821..54a584d78d2eb646679d1775ac08d9056f95ad07 100644
--- a/paddle/pten/kernels/hybird/cuda/cast_kernel_impl.h
+++ b/paddle/pten/kernels/hybird/cuda/cast_kernel_impl.h
@@ -15,13 +15,13 @@
 #pragma once
 #include "paddle/fluid/platform/device/gpu/gpu_helper.h"
 #include "paddle/fluid/platform/float16.h"
+#include "paddle/pten/backends/cuda/cuda_context.h"
 #include "paddle/pten/core/dense_tensor.h"
 
 #include "paddle/fluid/platform/aligned_vector.h"
 #include "paddle/fluid/platform/device/gpu/gpu_launch_config.h"
 namespace pten {
 namespace detail {
-using CUDAContext = paddle::platform::CUDADeviceContext;
 
 template <typename InT, typename OutT, int VecSize>
 __global__ void VecCastCUDAKernel(const InT* in, const int64_t N, OutT* out) {
diff --git a/paddle/pten/kernels/hybird/cuda/reduce/reduce.h b/paddle/pten/kernels/hybird/cuda/reduce/reduce.h
index 1e47726333bcd2e87007d63e274dee62be9c199a..c88965e6defea5cd2c6f042ab179877ef30f45ce 100644
--- a/paddle/pten/kernels/hybird/cuda/reduce/reduce.h
+++ b/paddle/pten/kernels/hybird/cuda/reduce/reduce.h
@@ -17,16 +17,13 @@
 // CUDA and HIP use same api
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 
+#include "paddle/pten/backends/cuda/cuda_context.h"
 #include "paddle/pten/common/scalar.h"
 #include "paddle/pten/core/dense_tensor.h"
-
-#include "paddle/fluid/platform/device_context.h"
 #include "paddle/pten/kernels/hybird/cuda/reduce/reduce_cuda_impl.h"
 
 namespace pten {
 
-using CUDAContext = paddle::platform::CUDADeviceContext;
-
 static inline std::vector<int64_t> GetReduceDim(
     const std::vector<int64_t>& dims, int dim_size, bool reduce_all) {
   std::vector<int64_t> reduce_dims;
diff --git a/paddle/pten/kernels/hybird/general/elementwise_base.h b/paddle/pten/kernels/hybird/general/elementwise_base.h
index 8c8a0342f6a1d2f8280b65bdc710f682ae0b67b7..827af86812ce7fa831dd9716297b475ca3b230bc 100644
--- a/paddle/pten/kernels/hybird/general/elementwise_base.h
+++ b/paddle/pten/kernels/hybird/general/elementwise_base.h
@@ -15,13 +15,13 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/fluid/platform/transform.h"
+#include "paddle/pten/backends/all_context.h"
 #include "paddle/pten/core/dense_tensor.h"
 
 namespace pten {
 namespace general {
 
 using DDim = paddle::framework::DDim;
-using CPUContext = paddle::platform::CPUDeviceContext;
 
 template <typename T, typename DeviceContext>
 class RowwiseTransformIterator;
@@ -131,7 +131,6 @@ class MidWiseTransformIterator<T, CPUContext>
 };
 
 #if defined(__NVCC__) || defined(__HIPCC__)
-using CUDAContext = paddle::platform::CUDADeviceContext;
 template <typename T>
 class RowwiseTransformIterator<T, CUDAContext>
     : public thrust::iterator_adaptor<RowwiseTransformIterator<T, CUDAContext>,
diff --git a/paddle/pten/kernels/hybird/transpose.cc b/paddle/pten/kernels/hybird/transpose.cc
index 73375a6a379fe9422620ac39879d27f5d8850bc1..d1d4350c931617b759d432299a1b219467739fe5 100644
--- a/paddle/pten/kernels/hybird/transpose.cc
+++ b/paddle/pten/kernels/hybird/transpose.cc
@@ -14,16 +14,16 @@
 
 #include "paddle/pten/kernels/hybird/transpose.h"
 #include "paddle/fluid/framework/ddim.h"
+#include "paddle/pten/backends/cpu/cpu_context.h"
 #include "paddle/pten/core/dense_tensor.h"
 
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/platform/bfloat16.h"
 #include "paddle/fluid/platform/complex.h"
-#include "paddle/fluid/platform/device_context.h"
 #include "paddle/fluid/platform/float16.h"
+
 namespace pten {
 namespace math {
-using CPUContext = paddle::platform::CPUDeviceContext;
 
 template <typename T>
 struct TransposeNormal<CPUContext, T> {
diff --git a/paddle/pten/kernels/hybird/transpose.cu b/paddle/pten/kernels/hybird/transpose.cu
index bf7a1409938a68963f9fefdc570e9da703c6fffa..ad2eda4edd99c112c2dfc4bd8df0d8217e3dd45a 100644
--- a/paddle/pten/kernels/hybird/transpose.cu
+++ b/paddle/pten/kernels/hybird/transpose.cu
@@ -14,6 +14,7 @@
 
 #include "paddle/fluid/framework/ddim.h"
 #include "paddle/fluid/memory/memcpy.h"
+#include "paddle/pten/backends/cuda/cuda_context.h"
 #include "paddle/pten/core/dense_tensor.h"
 #include "paddle/pten/kernels/hybird/math/cast_func.h"
 #include "paddle/pten/kernels/hybird/transpose.h"
@@ -21,13 +22,11 @@
 // See Note [ Why still include the fluid headers? ]
 #include "paddle/fluid/platform/bfloat16.h"
 #include "paddle/fluid/platform/complex.h"
-#include "paddle/fluid/platform/device_context.h"
 #include "paddle/fluid/platform/float16.h"
 
 namespace pten {
 
 namespace math {
-using CUDAContext = paddle::platform::CUDADeviceContext;
 
 #define REINTERPRET(T, DST_PTR, SRC_PTR) \
   T* DST_PTR = reinterpret_cast<T*>(SRC_PTR)
diff --git a/paddle/pten/kernels/xpu/manipulation.h b/paddle/pten/kernels/xpu/manipulation.h
index a9f57025e1e2c3e2cc215497589dfded17092d94..0b68ae419518e3196542999885707bdd734b14c9 100644
--- a/paddle/pten/kernels/xpu/manipulation.h
+++ b/paddle/pten/kernels/xpu/manipulation.h
@@ -16,17 +16,13 @@ limitations under the License. */
 
 #ifdef PADDLE_WITH_XPU
 
+#include "paddle/pten/backends/xpu/xpu_context.h"
 #include "paddle/pten/common/scalar_array.h"
 #include "paddle/pten/core/dense_tensor.h"
 #include "paddle/pten/core/kernel_registry.h"
 
-// See Note [ Why still include the fluid headers? ]
-#include "paddle/fluid/platform/device_context.h"
-
 namespace pten {
 
-using XPUContext = paddle::platform::XPUDeviceContext;
-
 template <typename T>
 void Flatten(const XPUContext& dev_ctx,
              const DenseTensor& x,