Merge pull request #1092 from gangliao/python_default

Check Python if system already equipped one

Merge pull request #1092 from gangliao/python_default
Check Python if system already equipped one
c016758f · gangliao · GitHub · 9eedd53e · 9a7df696 · c016758f
8 changed file
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,13 +21,14 @@ addons:
    packages:
      - gcc-4.8
      - g++-4.8
-      - wget
      - git
      - build-essential
      - libatlas-base-dev
      - python
      - python-pip
      - python2.7-dev
+      - python-numpy
+      - python-wheel
      - curl
      - swig
      - graphviz
@@ -48,8 +49,7 @@ before_install:
    fi
  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi
  - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi
-  - pip install --upgrade pip
-  - pip install wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker
+  - pip install numpy wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker
 script:
  - paddle/scripts/travis/main.sh
 notifications:

--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -28,21 +28,21 @@ include(system)
 include(simd)

 ###################### Configurations ############################
-option(WITH_DSO "Compile PaddlePaddle with dynamic linked libraries" ON)
-option(WITH_GPU "Compile PaddlePaddle with gpu" ${CUDA_FOUND})
-option(WITH_DOUBLE "Compile PaddlePaddle with double precision, otherwise use single precision" OFF)
-option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ${AVX_FOUND})
+option(WITH_GPU         "Compile PaddlePaddle with NVIDIA GPU"          ${CUDA_FOUND})
+option(WITH_AVX         "Compile PaddlePaddle with AVX intrinsics"      ${AVX_FOUND})
+option(WITH_DSO         "Compile PaddlePaddle with dynamic linked CUDA" ON)
+option(WITH_TESTING     "Compile PaddlePaddle with unit testing"        ON)
+option(WITH_SWIG_PY     "Compile PaddlePaddle with inference api"       ON)
+option(WITH_STYLE_CHECK "Compile PaddlePaddle with style check"         ON)
 option(WITH_PYTHON      "Compile PaddlePaddle with python interpreter"  ON)
-option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ON)
-option(WITH_RDMA "Compile PaddlePaddle with rdma support" OFF)
-option(WITH_TIMER "Compile PaddlePaddle use timer" OFF)
-option(WITH_PROFILER "Compile PaddlePaddle use gpu profiler" OFF)
-option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ON)
+option(WITH_DOUBLE      "Compile PaddlePaddle with double precision"    OFF)
+option(WITH_RDMA        "Compile PaddlePaddle with RDMA support"        OFF)
+option(WITH_TIMER       "Compile PaddlePaddle with stats timer"         OFF)
+option(WITH_PROFILER    "Compile PaddlePaddle with GPU profiler"        OFF)
 option(WITH_DOC         "Compile PaddlePaddle with documentation"       OFF)
-option(WITH_SWIG_PY "Compile PaddlePaddle with py PaddlePaddle prediction api" ON)
-option(ON_TRAVIS "Running test on travis-ci or not." OFF)
-option(ON_COVERALLS "Generating code coverage data on coveralls or not." OFF)
-option(COVERALLS_UPLOAD "Uploading the generated coveralls json." ON)
+option(ON_COVERALLS     "Compile PaddlePaddle with code coverage"       OFF)
+option(COVERALLS_UPLOAD "Package code coverage data to coveralls"       OFF)
+option(ON_TRAVIS        "Exclude special unit test on Travis CI"        OFF)

 include(external/zlib)      # download, build, install zlib
 include(external/gflags)    # download, build, install gflags
@@ -63,7 +63,6 @@ include(flags)              # set paddle compile flags
 include(cudnn)              # set cudnn libraries
 include(version)            # set PADDLE_VERSION
 include(coveralls)          # set code coverage
-include(python_module)      # set python module

 include(configure)          # add paddle env configuration


--- a/cmake/configure.cmake
+++ b/cmake/configure.cmake
@@ -12,6 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+if(NOT WITH_PYTHON)
+    add_definitions(-DPADDLE_NO_PYTHON)
+endif(NOT WITH_PYTHON)
+
 if(WITH_DSO)
    add_definitions(-DPADDLE_USE_DSO)
 endif(WITH_DSO)

--- a/cmake/external/python.cmake
+++ b/cmake/external/python.cmake
@@ -13,55 +13,60 @@
 # limitations under the License.

 INCLUDE(ExternalProject)
-
-
-##################################### PYTHON ########################################
-SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/python)
-SET(PYTHON_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/python)
-SET(_python_DIR ${PYTHON_INSTALL_DIR})
-
-IF(UNIX)
+INCLUDE(python_module)
+
+FIND_PACKAGE(PythonInterp 2.7)
+FIND_PACKAGE(PythonLibs 2.7)
+
+SET(py_env "")
+
+IF(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
+    find_python_module(pip REQUIRED)
+    find_python_module(numpy REQUIRED)
+    find_python_module(wheel REQUIRED)
+    find_python_module(google.protobuf REQUIRED)
+    FIND_PACKAGE(NumPy REQUIRED)
+ELSE(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
+    ##################################### PYTHON ########################################
+    SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/python)
+    SET(PYTHON_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/python)
+    SET(_python_DIR ${PYTHON_INSTALL_DIR})
+
+    IF(UNIX)
        SET(PYTHON_FOUND ON)
        SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include/python2.7" CACHE PATH "Python include dir" FORCE)
        SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/lib/libpython2.7.a" CACHE FILEPATH "Python library" FORCE)
        SET(PYTHON_EXECUTABLE ${PYTHON_INSTALL_DIR}/bin/python CACHE FILEPATH "Python executable" FORCE)
        SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/lib/python2.7/site-packages" CACHE PATH "Python site-packages path" FORCE)
-ELSEIF(WIN32)
+    ELSEIF(WIN32)
        SET(PYTHON_FOUND ON)
        SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include" CACHE PATH "Python include dir" FORCE)
        SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/libs/python27.lib" CACHE FILEPATH "Python library" FORCE)
        SET(PYTHON_EXECUTABLE "${PYTHON_INSTALL_DIR}/bin/python.exe" CACHE FILEPATH "Python executable" FORCE)
        SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/Lib/site-packages" CACHE PATH "Python site-packages path" FORCE)
-ELSE()
+    ELSE()
        MESSAGE(FATAL_ERROR "Unknown system !")
-ENDIF()
-
-SET(py_env
-    PATH=${PYTHON_INSTALL_DIR}/bin/:$ENV{PATH}
-    PYTHONHOME=${PYTHON_INSTALL_DIR}
-    PYTHONPATH=${PYTHON_INSTALL_DIR}/lib:${PYTHON_INSTALL_DIR}/lib/python2.7:${PY_SITE_PACKAGES_PATH})
-
-INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR})
+    ENDIF()

-IF(APPLE)
+    IF(APPLE)
        LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS
            -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON
            )
-ENDIF()
+    ENDIF()

-SET(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS)
+    SET(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS)

-# Force Python build to "Release".
-IF(CMAKE_CONFIGURATION_TYPES)
+    # Force Python build to "Release".
+    IF(CMAKE_CONFIGURATION_TYPES)
        SET(SAVED_CMAKE_CFG_INTDIR ${CMAKE_CFG_INTDIR})
        SET(CMAKE_CFG_INTDIR "Release")
-ELSE()
+    ELSE()
        LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS
            -DCMAKE_BUILD_TYPE:STRING=Release
            )
-ENDIF()
+    ENDIF()

-ExternalProject_Add(python
+    ExternalProject_Add(python
        ${EXTERNAL_PROJECT_LOG_ARGS}
        GIT_REPOSITORY    "https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git"
        PREFIX            ${PYTHON_SOURCES_DIR}
@@ -81,12 +86,17 @@ ExternalProject_Add(python
            ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS}
            ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS}
        DEPENDS zlib
-)
-####################################################################################
+    )

-##################################### SETUPTOOLS ###################################
-SET(SETUPTOOLS_SOURCES_DIR ${PYTHON_SOURCES_DIR}/setuptools)
-ExternalProject_Add(setuptools
+    SET(py_env
+        PATH=${PYTHON_INSTALL_DIR}/bin
+        PYTHONHOME=${PYTHON_INSTALL_DIR}
+        PYTHONPATH=${PYTHON_INSTALL_DIR}/lib:${PYTHON_INSTALL_DIR}/lib/python2.7:${PY_SITE_PACKAGES_PATH})
+    ####################################################################################
+
+    ##################################### SETUPTOOLS ###################################
+    SET(SETUPTOOLS_SOURCES_DIR ${PYTHON_SOURCES_DIR}/setuptools)
+    ExternalProject_Add(setuptools
        ${EXTERNAL_PROJECT_LOG_ARGS}
        PREFIX              ${SETUPTOOLS_SOURCES_DIR}
        URL                 "https://pypi.python.org/packages/source/s/setuptools/setuptools-18.3.2.tar.gz"
@@ -97,12 +107,12 @@ ExternalProject_Add(setuptools
        INSTALL_COMMAND     ""
        BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
        DEPENDS             python zlib
-)
-#####################################################################################
+    )
+    #####################################################################################

-##################################### SIX ###########################################
-SET(SIX_SOURCES_DIR ${PYTHON_SOURCES_DIR}/six)
-ExternalProject_Add(six
+    ##################################### SIX ###########################################
+    SET(SIX_SOURCES_DIR ${PYTHON_SOURCES_DIR}/six)
+    ExternalProject_Add(six
        ${EXTERNAL_PROJECT_LOG_ARGS}
        PREFIX              ${SIX_SOURCES_DIR}
        URL                 https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz
@@ -113,12 +123,12 @@ ExternalProject_Add(six
        INSTALL_COMMAND     ""
        BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
        DEPENDS             python setuptools
-)
-#####################################################################################
+    )
+    #####################################################################################

-##################################### CYTHON ########################################
-SET(CYTHON_SOURCES_DIR ${PYTHON_SOURCES_DIR}/cython)
-ExternalProject_Add(cython
+    ##################################### CYTHON ########################################
+    SET(CYTHON_SOURCES_DIR ${PYTHON_SOURCES_DIR}/cython)
+    ExternalProject_Add(cython
        ${EXTERNAL_PROJECT_LOG_ARGS}
        PREFIX                ${CYTHON_SOURCES_DIR}
        URL                   https://github.com/cython/cython/archive/0.25.2.tar.gz
@@ -130,19 +140,19 @@ ExternalProject_Add(cython
        INSTALL_COMMAND       ""
        BUILD_COMMAND         env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
        DEPENDS               python
-)
-####################################################################################
+    )
+    ####################################################################################

-##################################### NUMPY ########################################
-SET(NUMPY_SOURCES_DIR ${PYTHON_SOURCES_DIR}/numpy)
-SET(NUMPY_TAG_VERSION "v1.11.3")
-SET(NUMPY_VERSION "1.11.3")
+    ##################################### NUMPY ########################################
+    SET(NUMPY_SOURCES_DIR ${PYTHON_SOURCES_DIR}/numpy)
+    SET(NUMPY_TAG_VERSION "v1.11.3")
+    SET(NUMPY_VERSION "1.11.3")

-SET(EGG_NAME "")
-SET(PYTHON_NUMPY_INCLUDE_DIR "")
-IF(WIN32)
+    SET(EGG_NAME "")
+    SET(PYTHON_NUMPY_INCLUDE_DIR "")
+    IF(WIN32)
        SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}.egg")
-ELSE(WIN32)
+    ELSE(WIN32)
        IF(APPLE)
            SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}-${MACOS_VERSION}")
        ELSE(APPLE)
@@ -153,11 +163,9 @@ ELSE(WIN32)
        FOREACH(suffix x86_64 intel fat64 fat32 universal)
            LIST(APPEND PYTHON_NUMPY_INCLUDE_DIR ${PY_SITE_PACKAGES_PATH}/${EGG_NAME}-${suffix}.egg/numpy/core/include)
        ENDFOREACH()
-ENDIF(WIN32)
-
-INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR})
+    ENDIF(WIN32)

-ExternalProject_Add(numpy
+    ExternalProject_Add(numpy
        ${EXTERNAL_PROJECT_LOG_ARGS}
        GIT_REPOSITORY      https://github.com/numpy/numpy.git
        GIT_TAG             ${NUMPY_TAG_VERSION}
@@ -168,12 +176,12 @@ ExternalProject_Add(numpy
        INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
        BUILD_IN_SOURCE     1
        DEPENDS             python setuptools cython
-)
-####################################################################################
+    )
+    ####################################################################################

-##################################### WHEEL ########################################
-SET(WHEEL_SOURCES_DIR ${PYTHON_SOURCES_DIR}/wheel)
-ExternalProject_Add(wheel
+    ##################################### WHEEL ########################################
+    SET(WHEEL_SOURCES_DIR ${PYTHON_SOURCES_DIR}/wheel)
+    ExternalProject_Add(wheel
        ${EXTERNAL_PROJECT_LOG_ARGS}
        URL                 https://pypi.python.org/packages/source/w/wheel/wheel-0.29.0.tar.gz
        PREFIX              ${WHEEL_SOURCES_DIR}
@@ -183,12 +191,12 @@ ExternalProject_Add(wheel
        INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
        BUILD_IN_SOURCE     1
        DEPENDS             python setuptools
-)
-####################################################################################
+    )
+    ####################################################################################

-################################### PROTOBUF #######################################
-SET(PY_PROTOBUF_SOURCES_DIR ${PYTHON_SOURCES_DIR}/protobuf)
-ExternalProject_Add(python-protobuf
+    ################################### PROTOBUF #######################################
+    SET(PY_PROTOBUF_SOURCES_DIR ${PYTHON_SOURCES_DIR}/protobuf)
+    ExternalProject_Add(python-protobuf
        ${EXTERNAL_PROJECT_LOG_ARGS}
        URL                   https://pypi.python.org/packages/e0/b0/0a1b364fe8a7d177b4b7d4dca5b798500dc57a7273b93cca73931b305a6a/protobuf-3.1.0.post1.tar.gz
        URL_MD5               38b5fb160c768d2f8444d0c6d637ff91
@@ -196,9 +204,19 @@ ExternalProject_Add(python-protobuf
        BUILD_IN_SOURCE       1
        PATCH_COMMAND         ""
        CONFIGURE_COMMAND     ""
-    BUILD_COMMAND         env PATH=${PROTOBUF_INSTALL_DIR}/bin:$ENV{PATH} ${py_env} ${PYTHON_EXECUTABLE} setup.py build
-    INSTALL_COMMAND       env PATH=${PROTOBUF_INSTALL_DIR}/bin:$ENV{PATH} ${py_env} ${PYTHON_EXECUTABLE} setup.py install
+        BUILD_COMMAND         env ${py_env} ${PYTHON_EXECUTABLE} setup.py build
+        INSTALL_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
        DEPENDS               python setuptools six
-)
+    )
+    ####################################################################################
+
+    LIST(APPEND external_project_dependencies python setuptools six cython wheel python-protobuf numpy)
+
+ENDIF(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
+
+INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR})
+INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR})

-LIST(APPEND external_project_dependencies python setuptools six cython numpy wheel python-protobuf)
+MESSAGE("[Paddle] Python Executable: ${PYTHON_EXECUTABLE}")
+MESSAGE("[Paddle] Python Include: ${PYTHON_INCLUDE_DIRS}")
+MESSAGE("[Paddle] Python Libraries: ${PYTHON_LIBRARIES}")
--- a/paddle/api/CMakeLists.txt
+++ b/paddle/api/CMakeLists.txt
@@ -94,6 +94,7 @@ add_dependencies(python_api_wheel python_swig_sources
  paddle_cuda)

 if(WITH_TESTING)
+    IF(NOT PY_PIP_FOUND)
        SET(PIP_SOURCES_DIR ${PYTHON_SOURCES_DIR}/pip)
        ExternalProject_Add(pip
            ${EXTERNAL_PROJECT_LOG_ARGS}
@@ -106,5 +107,6 @@ if(WITH_TESTING)
            BUILD_IN_SOURCE     1
            DEPENDS python setuptools python_api_wheel
        )
+    ENDIF()
    add_subdirectory(test)
 endif()
--- a/paddle/scripts/travis/before_install.osx.sh
+++ b/paddle/scripts/travis/before_install.osx.sh
 #!/bin/bash
 brew update
 brew tap homebrew/science
-brew install openblas md5sha1sum 
+brew install python
+sudo pip install --upgrade protobuf
+brew install swig openblas md5sha1sum protobuf
--- a/paddle/scripts/travis/build_and_test.sh
+++ b/paddle/scripts/travis/build_and_test.sh
 #!/bin/bash
 source ./common.sh

-python -c 'import pip; print(pip.pep425tags.get_supported())'
-
-if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-  CMAKE_EXTRA="-DWITH_SWIG_PY=OFF"
-else
-  CMAKE_EXTRA="-DWITH_SWIG_PY=ON"
-fi
-
-cmake .. -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON -DON_COVERALLS=ON ${CMAKE_EXTRA}
-
 NPROC=1
 if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+  export PYTHONPATH=/opt/python/2.7.12/lib/python2.7/site-packages
+  export PYTHONHOME=/opt/python/2.7.12
+  export PATH=/opt/python/2.7.12/bin:${PATH}
+  cmake .. -DON_TRAVIS=ON -DON_COVERALLS=ON -DCOVERALLS_UPLOAD=ON
  NRPOC=`nproc`
  make -j $NPROC
  make coveralls
  sudo make install
 elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+  export PYTHONPATH=/usr/local/lib/python2.7/site-packages
+  cmake .. -DON_TRAVIS=ON -DON_COVERALLS=ON -DCOVERALLS_UPLOAD=ON -DWITH_SWIG_PY=ON
  NPROC=`sysctl -n hw.ncpu`
  make -j $NPROC
-  env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j $NPROC"
-  sudo make install
-  sudo paddle version
 fi
--- a/paddle/utils/PythonUtil.cpp.in
+++ b/paddle/utils/PythonUtil.cpp.in
@@ -195,8 +195,10 @@ extern const char enable_virtualenv_py[];
 }
 void initPython(int argc, char** argv) {
 #ifndef PADDLE_NO_PYTHON
-  char PythonHome[] = "@PYTHON_INSTALL_DIR@"; // NOLINT
-  Py_SetPythonHome(PythonHome);
+  char pyHome[] = "@PYTHON_INSTALL_DIR@"; // NOLINT
+  if (strlen(pyHome)) {
+    Py_SetPythonHome(pyHome);
+  }
  Py_SetProgramName(argv[0]);
  Py_Initialize();
  PySys_SetArgv(argc, argv);