diff --git a/CMakeLists.txt b/CMakeLists.txt
index e76512166fcaea5daf2a67d1259331b680f15b7c..3bdb3b73886a7309b2ef009f23381727ef6deafa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -54,6 +54,7 @@ option(WITH_C_API       "Compile PaddlePaddle with C-API(Prediction)"   OFF)
 option(WITH_GOLANG      "Compile PaddlePaddle with GOLANG"              OFF)
 option(GLIDE_INSTALL    "Download and install go dependencies "         ON)
 option(USE_NNPACK       "Compile PaddlePaddle with NNPACK library"      OFF)
+option(WITH_DISTRIBUTE  "Compile with grpc distributed support"         OFF)
 option(USE_EIGEN_FOR_BLAS   "Use matrix multiplication in Eigen"        OFF)
 
 # CMAKE_BUILD_TYPE
diff --git a/cmake/external/cares.cmake b/cmake/external/cares.cmake
index e05111ee18efc906e39bcb56fb1be3b3c3dff5d6..ac456933bd2260b2bbde2de78c486a5c0a1f5a96 100644
--- a/cmake/external/cares.cmake
+++ b/cmake/external/cares.cmake
@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 
-IF(MOBILE_INFERENCE)
+IF(MOBILE_INFERENCE OR NOT WITH_DISTRIBUTE)
     return()
 ENDIF()
 
diff --git a/cmake/external/grpc.cmake b/cmake/external/grpc.cmake
index 86122aec8c77f34756a37121582b92489d749d7f..abee6698e30b7e76ca42825ed225876bf2ba5ec0 100644
--- a/cmake/external/grpc.cmake
+++ b/cmake/external/grpc.cmake
@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 
-IF(MOBILE_INFERENCE)
+IF(MOBILE_INFERENCE OR NOT WITH_DISTRIBUTE)
     return()
 ENDIF()
 
diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt
index 937441b318095eadb9022c1d7578ad8aca2dadc8..54d3881b8cd7cdb20d8bf1cdfe9f2ecc0abe5c4d 100644
--- a/paddle/operators/CMakeLists.txt
+++ b/paddle/operators/CMakeLists.txt
@@ -212,6 +212,7 @@ set(DEPS_OPS
     send_op
     recv_op)
 
+if(WITH_DISTRIBUTE)
 add_subdirectory(detail)
 op_library(send_op SRCS send_op.cc DEPS sendrecvop_grpc grpc++_unsecure grpc_unsecure gpr cares zlib_target protobuf)
 set_source_files_properties(
@@ -225,6 +226,9 @@ set_source_files_properties(
     PROPERTIES
     COMPILE_FLAGS  "-Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
 
+cc_test(test_send_recv SRCS send_recv_op_test.cc DEPS send_op recv_op sum_op executor)
+endif()
+
 op_library(cond_op SRCS cond_op.cc DEPS framework_proto tensor operator net_op)
 op_library(cross_entropy_op DEPS cross_entropy)
 op_library(softmax_with_cross_entropy_op DEPS cross_entropy softmax)
@@ -275,4 +279,3 @@ if(WITH_GPU)
   cc_test(nccl_op_test SRCS nccl_op_test.cu.cc DEPS nccl_op gpu_info device_context)
 endif()
 cc_test(save_load_op_test SRCS save_load_op_test.cc DEPS save_op load_op)
-cc_test(test_send_recv SRCS send_recv_op_test.cc DEPS send_op recv_op sum_op executor)