diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake
index bfa56a9dbb40afa213d7e65879c93cc033cab745..6b76e3cc1d1a5905a027b08f7d08df4a47cc33b3 100644
--- a/cmake/inference_lib.cmake
+++ b/cmake/inference_lib.cmake
@@ -181,6 +181,13 @@ IF(WITH_XPU)
         DSTS ${dst_dir} ${dst_dir})
 ENDIF()
 
+IF(WITH_IPU)
+    set(dst_dir "${PADDLE_INFERENCE_INSTALL_DIR}/third_party/install/ipu")
+    copy(inference_lib_dist
+        SRCS ${CMAKE_BINARY_DIR}/paddle/fluid/platform/device/ipu/libpaddle_ipu.so
+        DSTS ${dst_dir})
+ENDIF()
+
 # CMakeCache Info
 copy(inference_lib_dist
         SRCS ${CMAKE_CURRENT_BINARY_DIR}/CMakeCache.txt
diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt
index d18ff6f6bfe2f0b04966af9e80bc40f3bebfc593..cbc61fc804397b1f0e4ae28fc792959bf5cfe82e 100644
--- a/paddle/fluid/operators/CMakeLists.txt
+++ b/paddle/fluid/operators/CMakeLists.txt
@@ -59,6 +59,10 @@ if(WITH_CINN)
     add_subdirectory(cinn)
 endif()
 
+if(WITH_IPU)
+    add_subdirectory(ipu)
+endif()
+
 SET(OP_HEADER_DEPS xxhash executor)
 
 if (WITH_GPU)
diff --git a/paddle/fluid/operators/ipu/CMakeLists.txt b/paddle/fluid/operators/ipu/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..66373d4b5f6b91914e9bb1f3ed5b7fdd5dec37ea
--- /dev/null
+++ b/paddle/fluid/operators/ipu/CMakeLists.txt
@@ -0,0 +1,3 @@
+if(WITH_IPU)
+  op_library(ipu_runtime_op DEPS ipu_backend)
+endif(WITH_IPU)
diff --git a/paddle/fluid/operators/ipu_runtime_op.h b/paddle/fluid/operators/ipu/ipu_runtime_op.cc
similarity index 55%
rename from paddle/fluid/operators/ipu_runtime_op.h
rename to paddle/fluid/operators/ipu/ipu_runtime_op.cc
index b6fc9ae98895d40d2e2d1c9eb02a63d200b0b1f8..3b6982d4b2b8e3fe29587e2e6cbbc16107326f78 100644
--- a/paddle/fluid/operators/ipu_runtime_op.h
+++ b/paddle/fluid/operators/ipu/ipu_runtime_op.cc
@@ -1,4 +1,4 @@
-//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//   Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,32 +12,29 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#pragma once
-#include <memory>
-#include <vector>
+#ifdef PADDLE_WITH_IPU
 
 #include "paddle/fluid/framework/op_registry.h"
-#ifdef PADDLE_WITH_IPU
-#include "paddle/fluid/framework/ipu/ipu_backend.h"
-#include "paddle/fluid/framework/tensor.h"
-#endif
+#include "paddle/fluid/platform/device/ipu/ipu_backend.h"
 
 namespace paddle {
 namespace operators {
 
-template <typename T>
-class IpuRuntimeKernel : public framework::OpKernel<T> {
+class IpuRuntimeOp : public framework::OperatorBase {
  public:
-  void Compute(const framework::ExecutionContext& ctx) const override {
-#ifdef PADDLE_WITH_IPU
-    auto ipu_backend = framework::ipu::IpuBackend::GetInstance();
-    if (!ipu_backend->DeviceIsAttached()) {
-      const platform::IPUDeviceContext& ipu_ctx =
-          reinterpret_cast<const platform::IPUDeviceContext&>(
-              ctx.device_context());
-      ipu_backend->AttachDevice(ipu_ctx.DeviceId());
-    }
+  IpuRuntimeOp(const std::string& type,
+               const framework::VariableNameMap& inputs,
+               const framework::VariableNameMap& outputs,
+               const framework::AttributeMap& attrs)
+      : OperatorBase(type, inputs, outputs, attrs) {}
 
+ private:
+  void RunImpl(const framework::Scope& scope,
+               const platform::Place& place) const {
+    auto ipu_backend = platform::ipu::IpuBackend::GetInstance();
+    auto* dev_ctx = platform::DeviceContextPool::Instance().Get(place);
+    framework::RuntimeContext runtime_ctx(inputs_, outputs_, scope);
+    framework::ExecutionContext ctx(*this, scope, *dev_ctx, runtime_ctx);
     auto inputs = ctx.MultiInput<framework::Tensor>("FeedList");
     auto outputs = ctx.MultiOutput<framework::Tensor>("FetchList");
     auto output_names = ctx.OutputNames("FetchList");
@@ -58,12 +55,24 @@ class IpuRuntimeKernel : public framework::OpKernel<T> {
                  << "(" << dim << ")";
       }
     }
-#else
-    PADDLE_THROW(platform::errors::PreconditionNotMet(
-        "Please compile WITH_IPU option to enable ipu_runtime op"));
-#endif
+  }
+};
+
+class IpuRuntimeOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+  void Make() override {
+    AddInput("FeedList", "FeedList of Graph").AsDuplicable();
+    AddOutput("FetchList", "FetchList of Graph").AsDuplicable();
+    AddComment(R"DOC(
+Run graph by PopART runtime.
+)DOC");
   }
 };
 
 }  // namespace operators
 }  // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OPERATOR(ipu_runtime, ops::IpuRuntimeOp, ops::IpuRuntimeOpMaker);
+
+#endif  // PADDLE_WITH_IPU
diff --git a/paddle/fluid/operators/ipu_runtime_op.cc b/paddle/fluid/operators/ipu_runtime_op.cc
deleted file mode 100644
index 4b473da00f3318135f194dd90151fbfb39315fee..0000000000000000000000000000000000000000
--- a/paddle/fluid/operators/ipu_runtime_op.cc
+++ /dev/null
@@ -1,62 +0,0 @@
-//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "paddle/fluid/operators/ipu_runtime_op.h"
-
-namespace paddle {
-namespace operators {
-
-class IpuRuntimeOp : public framework::OperatorWithKernel {
- public:
-  using framework::OperatorWithKernel::OperatorWithKernel;
-  void InferShape(framework::InferShapeContext* ctx) const override {}
-
- protected:
-  framework::OpKernelType GetExpectedKernelType(
-      const framework::ExecutionContext& ctx) const override {
-    return framework::OpKernelType(
-        framework::proto::VarType::Type(ctx.Attr<int>("dtype")),
-        ctx.device_context());
-  }
-};
-
-class IpuRuntimeOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
-  void Make() override {
-    AddInput("FeedList", "FeedList of Graph").AsDuplicable();
-    AddOutput("FetchList", "FetchList of Graph").AsDuplicable();
-    AddAttr<int>("dtype",
-                 "(int, default 5 (FP32)) "
-                 "Output data type")
-        .SetDefault(framework::proto::VarType::FP32);
-    AddComment(R"DOC(
-Run graph by PopART runtime.
-
-)DOC");
-  }
-};
-
-}  // namespace operators
-}  // namespace paddle
-
-namespace ops = paddle::operators;
-REGISTER_OPERATOR(ipu_runtime, ops::IpuRuntimeOp, ops::IpuRuntimeOpMaker);
-
-REGISTER_OP_IPU_KERNEL(ipu_runtime, ops::IpuRuntimeKernel<float>,
-                       ops::IpuRuntimeKernel<double>,
-                       ops::IpuRuntimeKernel<int>,
-                       ops::IpuRuntimeKernel<int64_t>,
-                       ops::IpuRuntimeKernel<bool>,
-                       ops::IpuRuntimeKernel<int8_t>,
-                       ops::IpuRuntimeKernel<paddle::platform::float16>);
diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt
index 8c1d3d0230a1020a7ac17ea916f78df6d4a18055..08ca575c2b936016e9c934535aac7960d5c10531 100644
--- a/paddle/fluid/pybind/CMakeLists.txt
+++ b/paddle/fluid/pybind/CMakeLists.txt
@@ -293,6 +293,10 @@ if(WITH_PYTHON)
     target_link_libraries(paddle_pybind ${ROCM_HIPRTC_LIB})
   endif()
 
+  if(WITH_IPU)
+    target_link_libraries(paddle_pybind paddle_ipu)
+  endif()
+
   get_property (os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
   target_link_libraries(paddle_pybind ${os_dependency_modules})
   add_dependencies(paddle_pybind op_function_generator_cmd)
diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc
index ad018944e431160afadb2720e2da891d7c5fec81..d3d7e5794e7b192d6aacee4adccfab554555187b 100644
--- a/paddle/fluid/pybind/pybind.cc
+++ b/paddle/fluid/pybind/pybind.cc
@@ -134,9 +134,10 @@ limitations under the License. */
 #endif
 
 #include "paddle/fluid/platform/cuda_graph_with_memory_pool.h"
+
 #ifdef PADDLE_WITH_IPU
-#include "paddle/fluid/platform/ipu/ipu_backend.h"
-#include "paddle/fluid/platform/ipu_info.h"
+#include "paddle/fluid/platform/device/ipu/ipu_backend.h"
+#include "paddle/fluid/platform/device/ipu/ipu_info.h"
 #endif
 
 #ifdef PADDLE_WITH_MLU
diff --git a/python/setup.py.in b/python/setup.py.in
index aee4e149b0afeeb4c33d617b7db2a19bf2c36a91..e8cc2914521f323fd50e491dab1f8e7eb1421a3b 100644
--- a/python/setup.py.in
+++ b/python/setup.py.in
@@ -519,6 +519,10 @@ if '${WITH_XPU_BKCL}' == 'ON':
     shutil.copy('${XPU_BKCL_LIB}', libs_path)
     package_data['paddle.libs']+=['${XPU_BKCL_LIB_NAME}']
 
+if '${WITH_IPU}' == 'ON':
+    shutil.copy('${PADDLE_IPU_LIB}', libs_path)
+    package_data['paddle.libs'] += ['libpaddle_ipu' + ext_name]
+
 # remove unused paddle/libs/__init__.py
 if os.path.isfile(libs_path+'/__init__.py'):
     os.remove(libs_path+'/__init__.py')