diff --git a/paddle/fluid/platform/dynload/CMakeLists.txt b/paddle/fluid/platform/dynload/CMakeLists.txt
index 2b63c81859d94eb62437439edbb71dc8c1dfbb42..07159d4a12ef4b628f7705ed206d3334be46dfc8 100644
--- a/paddle/fluid/platform/dynload/CMakeLists.txt
+++ b/paddle/fluid/platform/dynload/CMakeLists.txt
@@ -16,7 +16,6 @@ if (CUPTI_FOUND)
     list(APPEND CUDA_SRCS cupti.cc)
 endif(CUPTI_FOUND)
 nv_library(dynload_cuda SRCS ${CUDA_SRCS} DEPS dynamic_loader)
-configure_file(warpctc_lib_path.h.in ${CMAKE_CURRENT_BINARY_DIR}/warpctc_lib_path.h)
 cc_library(dynload_warpctc SRCS warpctc.cc DEPS dynamic_loader warpctc)
 if (WITH_MKLML)
     cc_library(dynload_mklml SRCS mklml.cc DEPS dynamic_loader mklml)
diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc
index 9aafc180b90c522ba8ee7508686279957ea97319..f7ad7a4f3ae3af96a13021a2335a84bc6261122d 100644
--- a/paddle/fluid/platform/dynload/dynamic_loader.cc
+++ b/paddle/fluid/platform/dynload/dynamic_loader.cc
@@ -20,7 +20,6 @@ limitations under the License. */
 #include "gflags/gflags.h"
 #include "glog/logging.h"
 #include "paddle/fluid/platform/dynload/cupti_lib_path.h"
-#include "paddle/fluid/platform/dynload/warpctc_lib_path.h"
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/platform/port.h"
 
@@ -34,8 +33,6 @@ DEFINE_string(cuda_dir, "",
               "libcurand. For instance, /usr/local/cuda/lib64. If default, "
               "dlopen will search cuda from LD_LIBRARY_PATH");
 
-DEFINE_string(warpctc_dir, "", "Specify path for loading libwarpctc.so.");
-
 DEFINE_string(nccl_dir, "",
               "Specify path for loading nccl library, such as libcublas, "
               "libcurand. For instance, /usr/local/cuda/lib64. If default, "
@@ -52,8 +49,15 @@ DEFINE_string(mklml_dir, "", "Specify path for loading libmklml_intel.so.");
 namespace paddle {
 namespace platform {
 namespace dynload {
+
+struct PathNode {
+  PathNode() {}
+  std::string path = "";
+};
+
 static constexpr char cupti_lib_path[] = CUPTI_LIB_PATH;
-static constexpr char warpctc_lib_path[] = WARPCTC_LIB_PATH;
+
+static PathNode s_py_site_pkg_path;
 
 #if defined(_WIN32) && defined(PADDLE_WITH_CUDA)
 static constexpr char* win_cublas_lib = "cublas64_" PADDLE_CUDA_BINVER ".dll";
@@ -78,6 +82,11 @@ static inline std::string join(const std::string& part1,
   return ret;
 }
 
+void SetPaddleLibPath(const std::string& py_site_pkg_path) {
+  s_py_site_pkg_path.path = py_site_pkg_path;
+  VLOG(3) << "Set paddle lib path : " << py_site_pkg_path;
+}
+
 static inline void* GetDsoHandleFromDefaultPath(const std::string& dso_path,
                                                 int dynload_flags) {
   VLOG(3) << "Try to find library: " << dso_path
@@ -214,9 +223,9 @@ void* GetCurandDsoHandle() {
 }
 
 void* GetWarpCTCDsoHandle() {
-  std::string warpctc_dir = warpctc_lib_path;
-  if (!FLAGS_warpctc_dir.empty()) {
-    warpctc_dir = FLAGS_warpctc_dir;
+  std::string warpctc_dir = "";
+  if (!s_py_site_pkg_path.path.empty()) {
+    warpctc_dir = s_py_site_pkg_path.path;
   }
 #if defined(__APPLE__) || defined(__OSX__)
   return GetDsoHandleFromSearchPath(warpctc_dir, "libwarpctc.dylib");
diff --git a/paddle/fluid/platform/dynload/dynamic_loader.h b/paddle/fluid/platform/dynload/dynamic_loader.h
index edb4c649addfaf941a00588395d9191038217979..d8bc884ee0987c24c6c6b08229c40b950c546a29 100644
--- a/paddle/fluid/platform/dynload/dynamic_loader.h
+++ b/paddle/fluid/platform/dynload/dynamic_loader.h
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #pragma once
+#include <string>
 
 namespace paddle {
 namespace platform {
@@ -33,6 +34,7 @@ void* GetNCCLDsoHandle();
 void* GetTensorRtDsoHandle();
 void* GetMKLMLDsoHandle();
 
+void SetPaddleLibPath(const std::string&);
 }  // namespace dynload
 }  // namespace platform
 }  // namespace paddle
diff --git a/paddle/fluid/platform/dynload/warpctc_lib_path.h.in b/paddle/fluid/platform/dynload/warpctc_lib_path.h.in
deleted file mode 100644
index dc5064f45735a9871b6b9f39fac06723c1b536f3..0000000000000000000000000000000000000000
--- a/paddle/fluid/platform/dynload/warpctc_lib_path.h.in
+++ /dev/null
@@ -1,17 +0,0 @@
-/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#pragma once
-
-#define WARPCTC_LIB_PATH "@WARPCTC_INSTALL_DIR@/lib/"
diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc
index b346c62d811472c2256a839eb29f257ad9010e31..9738601bd65640bfd178c0ff9af81dbe8a3dc3db 100644
--- a/paddle/fluid/pybind/pybind.cc
+++ b/paddle/fluid/pybind/pybind.cc
@@ -47,6 +47,7 @@ limitations under the License. */
 #include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
 #include "paddle/fluid/platform/cpu_helper.h"
 #include "paddle/fluid/platform/cpu_info.h"
+#include "paddle/fluid/platform/dynload/dynamic_loader.h"
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/platform/init.h"
 #include "paddle/fluid/platform/place.h"
@@ -197,6 +198,8 @@ PYBIND11_MODULE(core_noavx, m) {
   m.def("print_mem_usage",
         []() { return memory::allocation::GPUMemMonitor.PrintMemUsage(); });
 
+  m.def("_set_paddle_lib_path", &paddle::platform::dynload::SetPaddleLibPath);
+
   BindImperative(&m);
 
   py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
index dfdcc3ecd2aa6c8ffa19fc46740ba13b7f281501..b0bbda84c253e79e236bab0ed7369e8989b0dc2e 100755
--- a/paddle/scripts/paddle_build.sh
+++ b/paddle/scripts/paddle_build.sh
@@ -418,8 +418,6 @@ EOF
         #remove proxy here to fix dist error on mac
         export http_proxy=
         export https_proxy=
-        # TODO: jiabin need to refine this part when these tests fixed on mac
-        ctest --output-on-failure -j $2
         # make install should also be test when unittest
         make install -j 8
 
@@ -447,6 +445,9 @@ EOF
             pip3.7 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
         fi
 
+        # TODO: jiabin need to refine this part when these tests fixed on mac
+        ctest --output-on-failure -j $2
+
         paddle version
     fi
 }
diff --git a/python/paddle/fluid/core.py b/python/paddle/fluid/core.py
index a7295886647a8ef4e61b82b229cf7349d3336359..accffc17448a4f6f7c4b630d2149fcfed7c5135f 100644
--- a/python/paddle/fluid/core.py
+++ b/python/paddle/fluid/core.py
@@ -14,6 +14,7 @@
 
 from __future__ import print_function
 
+import site
 import sys
 import os
 
@@ -34,8 +35,8 @@ if os.path.exists(current_path + os.sep + 'core_noavx.' + core_suffix):
 try:
     if os.name == 'nt':
         third_lib_path = current_path + os.sep + '..' + os.sep + 'libs'
-        os.environ['path'] += ';' + third_lib_path
-        sys.path.append(third_lib_path)
+        os.environ['path'] = third_lib_path + ';' + os.environ['path']
+        sys.path.insert(0, third_lib_path)
 
 except ImportError as e:
     from .. import compat as cpt
@@ -175,6 +176,7 @@ if avx_supported():
         from .core_avx import _set_fuse_parameter_memory_size
         from .core_avx import _is_dygraph_debug_enabled
         from .core_avx import _dygraph_debug_level
+        from .core_avx import _set_paddle_lib_path
     except Exception as e:
         if has_avx_core:
             raise e
@@ -203,9 +205,29 @@ if load_noavx:
         from .core_noavx import _set_fuse_parameter_memory_size
         from .core_noavx import _is_dygraph_debug_enabled
         from .core_noavx import _dygraph_debug_level
+        from .core_noavx import _set_paddle_lib_path
     except Exception as e:
         if has_noavx_core:
             sys.stderr.write(
                 'Error: Can not import noavx core while this file exists ' +
                 current_path + os.sep + 'core_noavx.' + core_suffix + '\n')
         raise e
+
+
+# set paddle lib path
+def set_paddle_lib_path():
+    site_dirs = site.getsitepackages() if hasattr(
+        site,
+        'getsitepackages') else [x for x in sys.path if 'site-packages' in x]
+    for site_dir in site_dirs:
+        lib_dir = os.path.sep.join([site_dir, 'paddle', 'libs'])
+        if os.path.exists(lib_dir):
+            _set_paddle_lib_path(lib_dir)
+            return
+    if hasattr(site, 'USER_SITE'):
+        lib_dir = os.path.sep.join([site.USER_SITE, 'paddle', 'libs'])
+        if os.path.exists(lib_dir):
+            _set_paddle_lib_path(lib_dir)
+
+
+set_paddle_lib_path()
diff --git a/python/setup.py.in b/python/setup.py.in
index a392e230709168b88c38a2dfad162c1a8af60856..82b5f6d8ea10507a3733d2119c2e670e016294d6 100644
--- a/python/setup.py.in
+++ b/python/setup.py.in
@@ -3,13 +3,14 @@ import subprocess
 import os
 import re
 import shutil
+import sys
 class BinaryDistribution(Distribution):
     def has_ext_modules(foo):
         return True
 
 RC      = 0
 
-ext_name = '.dll' if os.name == 'nt' else '.so'
+ext_name = '.dll' if os.name == 'nt' else ('.dylib' if sys.platform == 'darwin' else '.so')
 
 def git_commit():
     try:
@@ -145,9 +146,9 @@ if '${CMAKE_SYSTEM_PROCESSOR}' not in ['arm', 'armv7-a', 'aarch64']:
 paddle_bins = ''
 if not '${WIN32}':
     paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/scripts/paddle']
-package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + (ext_name if os.name != 'nt' else '.pyd')]}
+package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + ('.so' if os.name != 'nt' else '.pyd')]}
 if '${HAS_NOAVX_CORE}' == 'ON':
-    package_data['paddle.fluid'] += ['core_noavx' + (ext_name if os.name != 'nt' else '.pyd')]
+    package_data['paddle.fluid'] += ['core_noavx' + ('.so' if os.name != 'nt' else '.pyd')]
 
 package_dir={
     '': '${PADDLE_BINARY_DIR}/python',
@@ -223,9 +224,9 @@ if '${CMAKE_BUILD_TYPE}' == 'Release':
     if os.name != 'nt':
         # only change rpath in Release mode, since in Debug mode, ${FLUID_CORE_NAME}.xx is too large to be changed.
         if "@APPLE@" == "1":
-            command = "install_name_tool -id \"@loader_path/../libs/\" ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + ext_name
+            command = "install_name_tool -id \"@loader_path/../libs/\" ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so'
         else:
-            command = "patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + ext_name
+            command = "patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so'
         if os.system(command) != 0:
             raise Exception("patch ${FLUID_CORE_NAME}.%s failed, command: %s" % (ext_name, command))
 
@@ -237,6 +238,8 @@ if os.name == 'nt':
         fix_package_dir[k] = v.replace('/', '\\')
     package_dir = fix_package_dir
     ext_modules = []
+elif sys.platform == 'darwin':
+    ext_modules = []
 
 setup(name='${PACKAGE_NAME}',
       version='${PADDLE_VERSION}',