diff --git a/CMakeLists.txt b/CMakeLists.txt
index 43bd4e0fcf86f2f3b39bebcaec296d40f66a18de..047db58cfdfd4e5d964a0465a7d1630da64900c6 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -333,6 +333,10 @@ else()
      endif()
 endif()
 
+if(WITH_HETERPS AND WITH_PSLIB)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
+endif()
+
 if(WITH_DISTRIBUTE)
     if(LINUX)
         set(WITH_GLOO ON CACHE STRING "Enable GLOO when compiling WITH_DISTRIBUTE=ON." FORCE)
diff --git a/cmake/external/gloo.cmake b/cmake/external/gloo.cmake
index 3b88e67f69fb37529c8d2785b2f2442a6a40ddb3..778d7c2a0ae29b89898533ea344f864a3db2036b 100644
--- a/cmake/external/gloo.cmake
+++ b/cmake/external/gloo.cmake
@@ -55,7 +55,7 @@ else()
       UPDATE_COMMAND        ""
       CONFIGURE_COMMAND     ""
       BUILD_COMMAND         mkdir -p ${GLOO_SOURCE_DIR}/build
-          && cd ${GLOO_SOURCE_DIR}/build && cmake .. && make
+          && cd ${GLOO_SOURCE_DIR}/build && cmake .. -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && make
           && mkdir -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/gloo
       INSTALL_COMMAND      ${CMAKE_COMMAND} -E copy ${GLOO_SOURCE_DIR}/build/gloo/libgloo.a ${GLOO_LIBRARY_DIR}
       COMMAND              ${CMAKE_COMMAND} -E copy_directory "${GLOO_SOURCE_DIR}/gloo/" "${GLOO_INCLUDE_DIR}/gloo"
diff --git a/cmake/flags.cmake b/cmake/flags.cmake
index 7495ee32bab95c73caaa7cbbc4905f9d9ab71dad..f90b71f9e60a861d8597ab0e6024b97ad5eccacb 100644
--- a/cmake/flags.cmake
+++ b/cmake/flags.cmake
@@ -202,6 +202,16 @@ if (APPLE)
     set (COMMON_FLAGS -Wno-deprecated-register)
 endif(APPLE)
 
+if(WITH_HETERPS AND WITH_PSLIB)
+    set(COMMON_FLAGS
+        -D_GLIBCXX_USE_CXX11_ABI=0
+        ${COMMON_FLAGS})
+
+    set(GPU_COMMON_FLAGS
+        -D_GLIBCXX_USE_CXX11_ABI=0
+        ${GPU_COMMON_FLAGS})
+endif()
+
 if(LINUX)
     set(GPU_COMMON_FLAGS
         -Wall
diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake
index 38e7d2039af8ad3c129bc8d7adf9e893c6d33fdb..ac3eff04d5383ecdf6c771babcaf3e6811600ac3 100644
--- a/cmake/third_party.cmake
+++ b/cmake/third_party.cmake
@@ -299,8 +299,10 @@ if(WITH_PSLIB)
 
         include(external/leveldb)
         list(APPEND third_party_deps extern_leveldb)
-        include(external/brpc)
-        list(APPEND third_party_deps extern_brpc)
+        if(NOT WITH_HETERPS)
+            include(external/brpc)
+            list(APPEND third_party_deps extern_brpc)
+        endif()
     endif()
 endif(WITH_PSLIB)
 
diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt
index bf6393544f715b75b53c514678d4354a7dc85636..27ba88b56f0bb27fc1be7b9db281e48ca871f585 100644
--- a/paddle/fluid/framework/CMakeLists.txt
+++ b/paddle/fluid/framework/CMakeLists.txt
@@ -118,7 +118,9 @@ cc_test(var_type_traits_test SRCS var_type_traits_test.cc DEPS var_type_traits)
 
 set(BRPC_DEPS "")
 if(WITH_PSLIB OR WITH_PSCORE)
-    set(BRPC_DEPS brpc ssl crypto)
+    if(NOT WITH_HETERPS)
+        set(BRPC_DEPS brpc ssl crypto)
+    endif()
     if(WITH_PSLIB_BRPC)
         set(BRPC_DEPS pslib_brpc)
     endif()
diff --git a/paddle/fluid/framework/device_worker.h b/paddle/fluid/framework/device_worker.h
index 7aadc856129a1302a0f349459636bda5e9456c1b..6c5adb5fcdf98538bfe3f6494ce2de250e78fdaf 100644
--- a/paddle/fluid/framework/device_worker.h
+++ b/paddle/fluid/framework/device_worker.h
@@ -61,7 +61,7 @@ bool CheckValidOutput(LoDTensor* tensor, size_t batch_size);
 
 class FleetWrapper;
 
-#ifdef PADDLE_WITH_PSLIB
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
 class HeterWrapper;
 #endif
 
@@ -359,7 +359,7 @@ class DownpourWorkerOpt : public DownpourWorker {
   uint64_t async_tid_ = 0;
 };
 
-#ifdef PADDLE_WITH_PSLIB
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
 class HeterCpuWorker : public HogwildWorker {
  public:
   HeterCpuWorker() {}
diff --git a/paddle/fluid/framework/device_worker_factory.cc b/paddle/fluid/framework/device_worker_factory.cc
index 8259d43cb9a4715508467d1e67edb08e6c836995..24834d39ce37c83a4b4c3bb94a7bffaefb22c17c 100644
--- a/paddle/fluid/framework/device_worker_factory.cc
+++ b/paddle/fluid/framework/device_worker_factory.cc
@@ -70,7 +70,7 @@ REGISTER_DEVICE_WORKER_CLASS(DownpourWorkerOpt);
 REGISTER_DEVICE_WORKER_CLASS(HeterSectionWorker);
 #endif
 
-#ifdef PADDLE_WITH_PSLIB
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
 REGISTER_DEVICE_WORKER_CLASS(HeterCpuWorker);
 #endif
 
diff --git a/paddle/fluid/framework/fleet/CMakeLists.txt b/paddle/fluid/framework/fleet/CMakeLists.txt
index 19d71076542bc0b549e6edce76a6bfa231561b6f..65214cb2591f607fc08e15459538c0ec39d2be23 100644
--- a/paddle/fluid/framework/fleet/CMakeLists.txt
+++ b/paddle/fluid/framework/fleet/CMakeLists.txt
@@ -2,7 +2,9 @@ if(WITH_PSLIB)
     if(WITH_PSLIB_BRPC)
         set(BRPC_DEPS pslib_brpc)
     else()
-        set(BRPC_DEPS brpc)
+        if(NOT WITH_HETERPS)
+            set(BRPC_DEPS brpc)
+        endif()
     endif(WITH_PSLIB_BRPC)
     cc_library(fleet_wrapper SRCS fleet_wrapper.cc DEPS framework_proto proto_desc op_registry variable_helper scope ${BRPC_DEPS} pslib)
 else()
diff --git a/paddle/fluid/framework/fleet/heter_wrapper.cc b/paddle/fluid/framework/fleet/heter_wrapper.cc
index 5b54aa03bb30a45bbbd9a550d197c5aef8633294..88c9a363d2789d3602c052cb156f20828ccddca8 100644
--- a/paddle/fluid/framework/fleet/heter_wrapper.cc
+++ b/paddle/fluid/framework/fleet/heter_wrapper.cc
@@ -27,7 +27,7 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "paddle/fluid/framework/fleet/heter_wrapper.h"
-#ifdef PADDLE_WITH_PSLIB
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
 #include "paddle/fluid/framework/device_worker.h"
 
 namespace paddle {
diff --git a/paddle/fluid/framework/fleet/heter_wrapper.h b/paddle/fluid/framework/fleet/heter_wrapper.h
index 4e529de077593777c1ab326db395febaefb9564a..76a21651bfac0ed425f54e679fe271b87ae91977 100644
--- a/paddle/fluid/framework/fleet/heter_wrapper.h
+++ b/paddle/fluid/framework/fleet/heter_wrapper.h
@@ -23,7 +23,7 @@ limitations under the License. */
 #include <unordered_map>
 #include <vector>
 
-#ifdef PADDLE_WITH_PSLIB
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
 #include "paddle/fluid/framework/heter_service.h"
 #include "paddle/fluid/framework/heter_util.h"
 #include "paddle/fluid/framework/scope.h"
diff --git a/paddle/fluid/framework/heter_service.h b/paddle/fluid/framework/heter_service.h
index 7e5bf138d9fa9270eef7b19e0b350301a2290ab7..9d0e3c50953bd87ddcf197befd8ed4e0443577f2 100644
--- a/paddle/fluid/framework/heter_service.h
+++ b/paddle/fluid/framework/heter_service.h
@@ -25,7 +25,7 @@ limitations under the License. */
 #include "paddle/fluid/framework/heter_service.pb.h"
 #include "paddle/fluid/framework/program_desc.h"
 #include "paddle/fluid/framework/scope.h"
-#ifdef PADDLE_WITH_PSLIB
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
 #include "brpc/channel.h"
 #include "brpc/controller.h"
 #include "brpc/server.h"
@@ -35,7 +35,7 @@ limitations under the License. */
 namespace paddle {
 namespace framework {
 
-#ifdef PADDLE_WITH_PSLIB
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
 typedef std::function<int(const HeterRequest*, HeterResponse*)>
     HeterServiceHandler;
 class DataFeed;
diff --git a/paddle/fluid/framework/heter_util.h b/paddle/fluid/framework/heter_util.h
index eb9f3040afe25e8d8095dea70c5a8c731718f8cb..b32b4df349f4e7a956d351c1511107ea90fe1689 100644
--- a/paddle/fluid/framework/heter_util.h
+++ b/paddle/fluid/framework/heter_util.h
@@ -14,7 +14,6 @@ limitations under the License. */
 
 #pragma once
 
-#ifdef PADDLE_WITH_PSLIB
 #include <fstream>
 #include <memory>
 #include <mutex>  // NOLINT
@@ -23,11 +22,12 @@ limitations under the License. */
 #include <unordered_map>  // NOLINT
 #include <unordered_set>  // NOLINT
 #include <vector>
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
 #include "bthread/bthread.h"
+#endif
 #include "paddle/fluid/framework/program_desc.h"
 #include "paddle/fluid/framework/scope.h"
 #include "paddle/fluid/platform/timer.h"
-
 namespace paddle {
 namespace framework {
 class DataFeed;
@@ -106,7 +106,7 @@ class HeterTask {
   double cpu_2_gpu_time{0};
   platform::Timer timeline;
 };
-#endif
+
 template <class T>
 class HeterObjectPool {
  public:
@@ -143,7 +143,7 @@ class HeterObjectPool {
   int num_{0};
 };
 
-#ifdef PADDLE_WITH_PSLIB
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
 struct BthreadMutextGuard {
   BthreadMutextGuard(bthread_mutex_t* rho) {
     mutex_ = rho;
@@ -328,6 +328,6 @@ class HeterList {
   int cap_;
   int size;
 };
+#endif
 }  // namespace framework
 }  // namespace paddle
-#endif
diff --git a/paddle/fluid/framework/hetercpu_worker.cc b/paddle/fluid/framework/hetercpu_worker.cc
index f50cc2769e9d63557b58deaa0e9a483544e42f82..75cc18887da9a14c30fb29db41d8ffc61fa98bcf 100644
--- a/paddle/fluid/framework/hetercpu_worker.cc
+++ b/paddle/fluid/framework/hetercpu_worker.cc
@@ -19,7 +19,7 @@ limitations under the License. */
 #include "paddle/fluid/platform/cpu_helper.h"
 #include "paddle/fluid/string/string_helper.h"
 
-#ifdef PADDLE_WITH_PSLIB
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
 
 #if defined _WIN32 || defined __APPLE__
 #else
diff --git a/paddle/fluid/framework/heterxpu_trainer.cc b/paddle/fluid/framework/heterxpu_trainer.cc
index 01430781c64cdeeba114f40d0a5ccd5d9b435485..e47681820e282a41d244d6c50b1e64fd0fe65839 100644
--- a/paddle/fluid/framework/heterxpu_trainer.cc
+++ b/paddle/fluid/framework/heterxpu_trainer.cc
@@ -24,7 +24,7 @@ limitations under the License. */
 #include "paddle/fluid/framework/fleet/heter_wrapper.h"
 #include "paddle/fluid/framework/trainer.h"
 #if (defined PADDLE_WITH_CUDA || defined PADDLE_WITH_XPU) && \
-    (defined PADDLE_WITH_PSLIB)
+    (defined PADDLE_WITH_PSLIB) && (!defined(PADDLE_WITH_HETERPS))
 #ifdef PADDLE_WITH_CUDA
 #include "paddle/fluid/platform/cuda_device_guard.h"
 #endif
diff --git a/paddle/fluid/framework/trainer.h b/paddle/fluid/framework/trainer.h
index 6c97c7fefb184b033773f835cbf5b48014f4aa6b..40a8aa459dafeb47d2dda8d876de3a403a838090 100644
--- a/paddle/fluid/framework/trainer.h
+++ b/paddle/fluid/framework/trainer.h
@@ -155,7 +155,7 @@ class DistMultiTrainer : public MultiTrainer {
 
 #if (defined PADDLE_WITH_CUDA || defined PADDLE_WITH_HIP || \
      defined PADDLE_WITH_XPU) &&                            \
-    (defined PADDLE_WITH_PSLIB)
+    (defined PADDLE_WITH_PSLIB) && (!defined(PADDLE_WITH_HETERPS))
 class HeterServiceContext {
  public:
   HeterServiceContext() {}
diff --git a/paddle/fluid/framework/trainer_factory.cc b/paddle/fluid/framework/trainer_factory.cc
index 6f003c2f497b68527768af7fefd8ff69e9c23d3f..f189d0213da881399f5ef63f14cd7e16a86d8799 100644
--- a/paddle/fluid/framework/trainer_factory.cc
+++ b/paddle/fluid/framework/trainer_factory.cc
@@ -73,7 +73,7 @@ REGISTER_TRAINER_CLASS(HeterPipelineTrainer);
 
 #if (defined PADDLE_WITH_CUDA || defined PADDLE_WITH_HIP || \
      defined PADDLE_WITH_XPU) &&                            \
-    (defined PADDLE_WITH_PSLIB)
+    (defined PADDLE_WITH_PSLIB) && (!defined(PADDLE_WITH_HETERPS))
 REGISTER_TRAINER_CLASS(HeterXpuTrainer);
 #endif
 #if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL) && \
diff --git a/paddle/fluid/pybind/heter_wrapper_py.cc b/paddle/fluid/pybind/heter_wrapper_py.cc
index 0603cd3faae1d7007ec31813373019172baf6eb7..0b3565d94a253eb862f8a8860870e06c42347ff3 100644
--- a/paddle/fluid/pybind/heter_wrapper_py.cc
+++ b/paddle/fluid/pybind/heter_wrapper_py.cc
@@ -33,7 +33,7 @@ namespace py = pybind11;
 
 namespace paddle {
 namespace pybind {
-#ifdef PADDLE_WITH_PSLIB
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
 void BindHeterWrapper(py::module* m) {
   py::class_<framework::HeterWrapper, std::shared_ptr<framework::HeterWrapper>>(
       *m, "Heter")
diff --git a/paddle/fluid/pybind/heter_wrapper_py.h b/paddle/fluid/pybind/heter_wrapper_py.h
index 047c1f359ea8397998b967afbe30f5c0ff8d6170..b9bfcf070f5b69e344edc86d8d450f28b88d767e 100644
--- a/paddle/fluid/pybind/heter_wrapper_py.h
+++ b/paddle/fluid/pybind/heter_wrapper_py.h
@@ -22,7 +22,7 @@ namespace py = pybind11;
 namespace paddle {
 namespace pybind {
 
-#ifdef PADDLE_WITH_PSLIB
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
 void BindHeterWrapper(py::module* m);
 #endif
 }  // namespace pybind
diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc
index d3d7e5794e7b192d6aacee4adccfab554555187b..3e94ccfd497569036290d437353128d2183ffa20 100644
--- a/paddle/fluid/pybind/pybind.cc
+++ b/paddle/fluid/pybind/pybind.cc
@@ -3676,7 +3676,7 @@ All parameter, weight, gradient are variables in Paddle.
   BindFleetWrapper(&m);
   BindIO(&m);
 
-#ifdef PADDLE_WITH_PSLIB
+#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
   BindHeterWrapper(&m);
 #endif
 #ifdef PADDLE_WITH_HETERPS