diff --git a/CMakeLists.txt b/CMakeLists.txt
index f7824b1066473104213a00011242a4f47deab246..36b6168076d04acf9e7f3e50815fcd11e6668557 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -54,6 +54,7 @@ option(WITH_C_API       "Compile PaddlePaddle with C-API(Prediction)"   OFF)
 option(WITH_GOLANG      "Compile PaddlePaddle with GOLANG"              OFF)
 option(GLIDE_INSTALL    "Download and install go dependencies "         ON)
 option(USE_NNPACK       "Compile PaddlePaddle with NNPACK library"      OFF)
+option(WITH_DISTRIBUTE  "Compile with grpc distributed support"         OFF)
 option(USE_EIGEN_FOR_BLAS   "Use matrix multiplication in Eigen"        OFF)
 
 # CMAKE_BUILD_TYPE
diff --git a/cmake/external/cares.cmake b/cmake/external/cares.cmake
index e05111ee18efc906e39bcb56fb1be3b3c3dff5d6..ac456933bd2260b2bbde2de78c486a5c0a1f5a96 100644
--- a/cmake/external/cares.cmake
+++ b/cmake/external/cares.cmake
@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 
-IF(MOBILE_INFERENCE)
+IF(MOBILE_INFERENCE OR NOT WITH_DISTRIBUTE)
     return()
 ENDIF()
 
diff --git a/cmake/external/grpc.cmake b/cmake/external/grpc.cmake
index 86122aec8c77f34756a37121582b92489d749d7f..abee6698e30b7e76ca42825ed225876bf2ba5ec0 100644
--- a/cmake/external/grpc.cmake
+++ b/cmake/external/grpc.cmake
@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 
-IF(MOBILE_INFERENCE)
+IF(MOBILE_INFERENCE OR NOT WITH_DISTRIBUTE)
     return()
 ENDIF()
 
diff --git a/cmake/generic.cmake b/cmake/generic.cmake
index 9cf256fb6d5336b92f94198517f700e77a9330b3..66c8e3ad7ef7c80c1f388c25983425a0db5c0220 100644
--- a/cmake/generic.cmake
+++ b/cmake/generic.cmake
@@ -505,12 +505,12 @@ function(grpc_library TARGET_NAME)
   set_source_files_properties(
     ${grpc_grpc_srcs}
     PROPERTIES
-    COMPILE_FLAGS  "-Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
+    COMPILE_FLAGS  "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
   cc_library("${TARGET_NAME}_grpc" SRCS "${grpc_grpc_srcs}")
 
   set_source_files_properties(
     ${grpc_library_SRCS}
     PROPERTIES
-    COMPILE_FLAGS  "-Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
+    COMPILE_FLAGS  "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
   cc_library("${TARGET_NAME}" SRCS "${grpc_library_SRCS}" DEPS "${TARGET_NAME}_grpc" "${TARGET_NAME}_proto" "${grpc_library_DEPS}")
 endfunction()
diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt
index 937441b318095eadb9022c1d7578ad8aca2dadc8..38b89b9eb108d73c3374360a81c6ed28502bfdc5 100644
--- a/paddle/operators/CMakeLists.txt
+++ b/paddle/operators/CMakeLists.txt
@@ -212,18 +212,22 @@ set(DEPS_OPS
     send_op
     recv_op)
 
+if(WITH_DISTRIBUTE)
 add_subdirectory(detail)
 op_library(send_op SRCS send_op.cc DEPS sendrecvop_grpc grpc++_unsecure grpc_unsecure gpr cares zlib_target protobuf)
 set_source_files_properties(
     send_op.cc
     PROPERTIES
-    COMPILE_FLAGS  "-Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
+    COMPILE_FLAGS  "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
 
 op_library(recv_op SRCS recv_op.cc DEPS sendrecvop_grpc grpc++_unsecure grpc_unsecure gpr cares zlib_target protobuf)
 set_source_files_properties(
     recv_op.cc
     PROPERTIES
-    COMPILE_FLAGS  "-Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
+    COMPILE_FLAGS  "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
+
+cc_test(test_send_recv SRCS send_recv_op_test.cc DEPS send_op recv_op sum_op executor)
+endif()
 
 op_library(cond_op SRCS cond_op.cc DEPS framework_proto tensor operator net_op)
 op_library(cross_entropy_op DEPS cross_entropy)
@@ -275,4 +279,3 @@ if(WITH_GPU)
   cc_test(nccl_op_test SRCS nccl_op_test.cu.cc DEPS nccl_op gpu_info device_context)
 endif()
 cc_test(save_load_op_test SRCS save_load_op_test.cc DEPS save_op load_op)
-cc_test(test_send_recv SRCS send_recv_op_test.cc DEPS send_op recv_op sum_op executor)
diff --git a/paddle/platform/enforce.h b/paddle/platform/enforce.h
index 97338a4ce68da46ead7d66f7e01a3d5c8623b3c4..5abd4d4a345ed2750231841325f2b19a2ee8c4c9 100644
--- a/paddle/platform/enforce.h
+++ b/paddle/platform/enforce.h
@@ -244,7 +244,7 @@ inline void throw_on_error(T e) {
 
 #define __PADDLE_BINARY_COMPARE(__VAL0, __VAL1, __CMP, __INV_CMP, ...)  \
   do {                                                                  \
-    if (!UNLIKELY((__VAL0)__CMP(__VAL1))) {                             \
+    if (UNLIKELY(!((__VAL0)__CMP(__VAL1)))) {                           \
       PADDLE_THROW("enforce %s " #__CMP " %s failed, %s " #__INV_CMP    \
                    " %s\n%s",                                           \
                    #__VAL0, #__VAL1, paddle::string::to_string(__VAL0), \
diff --git a/paddle/platform/gpu_info.cc b/paddle/platform/gpu_info.cc
index 36b216d872138d49bfd5ab6e3499d15d49ebd0ca..63a33517086ec96711e610b766d19431e084e047 100644
--- a/paddle/platform/gpu_info.cc
+++ b/paddle/platform/gpu_info.cc
@@ -75,15 +75,19 @@ size_t GpuMaxChunkSize() {
   GpuMemoryUsage(available, total);
 
   // Reserving the rest memory for page tables, etc.
-  size_t reserving = (1 - FLAGS_fraction_of_gpu_memory_to_use) * total;
+  size_t reserving = 0.05 * total;
 
   // If available less than minimum chunk size, no usable memory exists.
-  available = std::max(available, GpuMinChunkSize()) - GpuMinChunkSize();
+  available =
+      std::max(std::max(available, GpuMinChunkSize()) - GpuMinChunkSize(),
+               reserving) -
+      reserving;
 
-  // If available less than reserving, no usable memory exists.
-  size_t usable = std::max(available, reserving) - reserving;
+  size_t allocating = FLAGS_fraction_of_gpu_memory_to_use * total;
 
-  return usable;
+  PADDLE_ENFORCE_LT(allocating, available);
+
+  return allocating;
 }
 
 void GpuMemcpyAsync(void *dst, const void *src, size_t count,
diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh
index 502637c881208e53dd832a9759b3873ef1988395..fbd0b6b07876451ad973eb98bbff822a2a58db43 100644
--- a/paddle/scripts/docker/build.sh
+++ b/paddle/scripts/docker/build.sh
@@ -36,6 +36,7 @@ function cmake_gen() {
         ${PYTHON_FLAGS}
         -DWITH_DOC=OFF
         -DWITH_GPU=${WITH_GPU:-OFF}
+        -DWITH_DISTRIBUTE=${WITH_DISTRIBUTE:-OFF}
         -DWITH_MKL=${WITH_MKL:-ON}
         -DWITH_AVX=${WITH_AVX:-OFF}
         -DWITH_GOLANG=${WITH_GOLANG:-ON}
@@ -57,6 +58,7 @@ EOF
         ${PYTHON_FLAGS} \
         -DWITH_DOC=OFF \
         -DWITH_GPU=${WITH_GPU:-OFF} \
+        -DWITH_DISTRIBUTE=${WITH_DISTRIBUTE:-OFF} \
         -DWITH_MKL=${WITH_MKL:-ON} \
         -DWITH_AVX=${WITH_AVX:-OFF} \
         -DWITH_GOLANG=${WITH_GOLANG:-ON} \