diff --git a/paddle/fluid/operators/math/math_function.cc b/paddle/fluid/operators/math/math_function.cc
index 8c7437e4b5e720af28440119f7781d7940638525..1da8c89a6d1a868893a5ec76c20c5064c01be944 100644
--- a/paddle/fluid/operators/math/math_function.cc
+++ b/paddle/fluid/operators/math/math_function.cc
@@ -99,13 +99,7 @@ struct TransposeNormal<platform::CPUDeviceContext, T> {
         out_ptr[out_idx] = in_ptr[in_idx];
       }
     };
-    double cost_per_iteration =
-        rank * (Eigen::TensorOpCost::DivCost<int64_t>() +
-                2 * Eigen::TensorOpCost::MulCost<int64_t>() +
-                2 * Eigen::TensorOpCost::AddCost<int64_t>());
-    Eigen::TensorOpCost cost(sizeof(T), sizeof(T), cost_per_iteration);
-    auto* cpu_device = context.eigen_pool_device();
-    cpu_device->parallelFor(out->numel(), cost, std::move(transpose_helper));
+    transpose_helper(0, out->numel());
   }
 };
 
diff --git a/paddle/fluid/platform/device_context.cc b/paddle/fluid/platform/device_context.cc
index 2e1517aa79ef737d1b96d9f1ec6b1193fe02f707..29982c13c8ca88bc8b4a168f92e4116a283a97e8 100644
--- a/paddle/fluid/platform/device_context.cc
+++ b/paddle/fluid/platform/device_context.cc
@@ -12,7 +12,6 @@ limitations under the License. */
 #include "paddle/fluid/platform/device_context.h"
 #include <set>
 #include <string>
-#include <thread>  //NOLINT
 #include <unordered_set>
 #include <vector>
 
@@ -24,7 +23,6 @@ limitations under the License. */
 #endif
 
 #include "glog/logging.h"
-#include "unsupported/Eigen/CXX11/ThreadPool"
 
 namespace paddle {
 namespace memory {
@@ -133,32 +131,16 @@ DeviceContextPool::DeviceContextPool(
 
 CPUDeviceContext::CPUDeviceContext() {
   eigen_device_.reset(new Eigen::DefaultDevice());
-  InitPoolDevice();
 }
 
 CPUDeviceContext::CPUDeviceContext(CPUPlace place) : place_(place) {
   eigen_device_.reset(new Eigen::DefaultDevice());
-  InitPoolDevice();
-}
-
-void CPUDeviceContext::InitPoolDevice() {
-  using EigenEnv = Eigen::StlThreadEnvironment;
-  using EigenThreadPool = Eigen::ThreadPoolTempl<EigenEnv>;
-  // int num_threads = std::thread::hardware_concurrency();
-  int num_threads = 1;
-  eigen_threadpool_.reset(new EigenThreadPool(num_threads));
-  eigen_pool_device_.reset(
-      new Eigen::ThreadPoolDevice(eigen_threadpool_.get(), num_threads));
 }
 
 Eigen::DefaultDevice* CPUDeviceContext::eigen_device() const {
   return eigen_device_.get();
 }
 
-Eigen::ThreadPoolDevice* CPUDeviceContext::eigen_pool_device() const {
-  return eigen_pool_device_.get();
-}
-
 Place CPUDeviceContext::GetPlace() const { return place_; }
 
 #ifdef PADDLE_WITH_XPU
diff --git a/paddle/fluid/platform/device_context.h b/paddle/fluid/platform/device_context.h
index 074106f3f205121c145fa98e2b766c1cb8354bc3..620e2d41c13aff8c70b0b96487cf8d541699d64b 100644
--- a/paddle/fluid/platform/device_context.h
+++ b/paddle/fluid/platform/device_context.h
@@ -43,7 +43,6 @@ limitations under the License. */
 #ifdef PADDLE_WITH_CUDA
 #include "paddle/fluid/platform/stream/cuda_stream.h"
 #endif
-#define EIGEN_USE_THREADS
 #include "unsupported/Eigen/CXX11/Tensor"
 
 namespace Eigen {
@@ -73,17 +72,11 @@ class CPUDeviceContext : public DeviceContext {
 
   Eigen::DefaultDevice* eigen_device() const;
 
-  Eigen::ThreadPoolDevice* eigen_pool_device() const;
-
   Place GetPlace() const override;
 
-  inline void InitPoolDevice();
-
  private:
   CPUPlace place_;
   std::unique_ptr<Eigen::DefaultDevice> eigen_device_;
-  std::unique_ptr<Eigen::ThreadPoolDevice> eigen_pool_device_;
-  std::unique_ptr<Eigen::ThreadPool> eigen_threadpool_;
 };
 
 template <typename Place>