Check python if system already equipped one

.travis.yml

@@ -21,13 +21,14 @@ addons:
     packages:
       - gcc-4.8
       - g++-4.8
-      - wget
       - git
       - build-essential
       - libatlas-base-dev
       - python
       - python-pip
       - python2.7-dev
+      - python-numpy
+      - python-wheel
       - curl
       - swig
       - graphviz
@@ -48,8 +49,7 @@ before_install:
     fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi
   - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi
-  - pip install --upgrade pip
-  - pip install wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker
+  - pip install numpy wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker
 script:
   - paddle/scripts/travis/main.sh
 notifications:

cmake/configure.cmake

@@ -12,6 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+if(NOT WITH_PYTHON)
+    add_definitions(-DPADDLE_NO_PYTHON)
+endif(NOT WITH_PYTHON)
+
 if(WITH_DSO)
     add_definitions(-DPADDLE_USE_DSO)
 endif(WITH_DSO)

cmake/external/python.cmake

@@ -13,55 +13,60 @@
 # limitations under the License.
 
 INCLUDE(ExternalProject)
-
-
-##################################### PYTHON ########################################
-SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/python)
-SET(PYTHON_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/python)
-SET(_python_DIR ${PYTHON_INSTALL_DIR})
-
-IF(UNIX)
+INCLUDE(python_module)
+
+FIND_PACKAGE(PythonInterp 2.7)
+FIND_PACKAGE(PythonLibs 2.7)
+
+SET(py_env PATH=${PATH} PYTHONHOME=${PYTHONHOME} PYTHONPATH=${PYTHONPATH})
+
+IF(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
+    find_python_module(pip REQUIRED)
+    find_python_module(numpy REQUIRED)
+    find_python_module(wheel REQUIRED)
+    find_python_module(google.protobuf REQUIRED)
+    FIND_PACKAGE(NumPy REQUIRED)
+ELSE(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
+    ##################################### PYTHON ########################################
+    SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/python)
+    SET(PYTHON_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/python)
+    SET(_python_DIR ${PYTHON_INSTALL_DIR})
+
+    IF(UNIX)
         SET(PYTHON_FOUND ON)
         SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include/python2.7" CACHE PATH "Python include dir" FORCE)
         SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/lib/libpython2.7.a" CACHE FILEPATH "Python library" FORCE)
         SET(PYTHON_EXECUTABLE ${PYTHON_INSTALL_DIR}/bin/python CACHE FILEPATH "Python executable" FORCE)
         SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/lib/python2.7/site-packages" CACHE PATH "Python site-packages path" FORCE)
-ELSEIF(WIN32)
+    ELSEIF(WIN32)
         SET(PYTHON_FOUND ON)
         SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include" CACHE PATH "Python include dir" FORCE)
         SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/libs/python27.lib" CACHE FILEPATH "Python library" FORCE)
         SET(PYTHON_EXECUTABLE "${PYTHON_INSTALL_DIR}/bin/python.exe" CACHE FILEPATH "Python executable" FORCE)
         SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/Lib/site-packages" CACHE PATH "Python site-packages path" FORCE)
-ELSE()
+    ELSE()
         MESSAGE(FATAL_ERROR "Unknown system !")
-ENDIF()
-
-SET(py_env
-    PATH=${PYTHON_INSTALL_DIR}/bin/:$ENV{PATH}
-    PYTHONHOME=${PYTHON_INSTALL_DIR}
-    PYTHONPATH=${PYTHON_INSTALL_DIR}/lib:${PYTHON_INSTALL_DIR}/lib/python2.7:${PY_SITE_PACKAGES_PATH})
-
-INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR})
+    ENDIF()
 
-IF(APPLE)
+    IF(APPLE)
         LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS
             -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON
             )
-ENDIF()
+    ENDIF()
 
-SET(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS)
+    SET(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS)
 
-# Force Python build to "Release".
-IF(CMAKE_CONFIGURATION_TYPES)
+    # Force Python build to "Release".
+    IF(CMAKE_CONFIGURATION_TYPES)
         SET(SAVED_CMAKE_CFG_INTDIR ${CMAKE_CFG_INTDIR})
         SET(CMAKE_CFG_INTDIR "Release")
-ELSE()
+    ELSE()
         LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS
             -DCMAKE_BUILD_TYPE:STRING=Release
             )
-ENDIF()
+    ENDIF()
 
-ExternalProject_Add(python
+    ExternalProject_Add(python
         ${EXTERNAL_PROJECT_LOG_ARGS}
         GIT_REPOSITORY    "https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git"
         PREFIX            ${PYTHON_SOURCES_DIR}
@@ -81,12 +86,17 @@ ExternalProject_Add(python
             ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS}
             ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS}
         DEPENDS zlib
-)
-####################################################################################
+    )
 
-##################################### SETUPTOOLS ###################################
-SET(SETUPTOOLS_SOURCES_DIR ${PYTHON_SOURCES_DIR}/setuptools)
-ExternalProject_Add(setuptools
+    SET(py_env
+        PATH=${PYTHON_INSTALL_DIR}/bin
+        PYTHONHOME=${PYTHON_INSTALL_DIR}
+        PYTHONPATH=${PYTHON_INSTALL_DIR}/lib:${PYTHON_INSTALL_DIR}/lib/python2.7:${PY_SITE_PACKAGES_PATH})
+    ####################################################################################
+
+    ##################################### SETUPTOOLS ###################################
+    SET(SETUPTOOLS_SOURCES_DIR ${PYTHON_SOURCES_DIR}/setuptools)
+    ExternalProject_Add(setuptools
         ${EXTERNAL_PROJECT_LOG_ARGS}
         PREFIX              ${SETUPTOOLS_SOURCES_DIR}
         URL                 "https://pypi.python.org/packages/source/s/setuptools/setuptools-18.3.2.tar.gz"
@@ -97,12 +107,12 @@ ExternalProject_Add(setuptools
         INSTALL_COMMAND     ""
         BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
         DEPENDS             python zlib
-)
-#####################################################################################
+    )
+    #####################################################################################
 
-##################################### SIX ###########################################
-SET(SIX_SOURCES_DIR ${PYTHON_SOURCES_DIR}/six)
-ExternalProject_Add(six
+    ##################################### SIX ###########################################
+    SET(SIX_SOURCES_DIR ${PYTHON_SOURCES_DIR}/six)
+    ExternalProject_Add(six
         ${EXTERNAL_PROJECT_LOG_ARGS}
         PREFIX              ${SIX_SOURCES_DIR}
         URL                 https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz
@@ -113,12 +123,12 @@ ExternalProject_Add(six
         INSTALL_COMMAND     ""
         BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
         DEPENDS             python setuptools
-)
-#####################################################################################
+    )
+    #####################################################################################
 
-##################################### CYTHON ########################################
-SET(CYTHON_SOURCES_DIR ${PYTHON_SOURCES_DIR}/cython)
-ExternalProject_Add(cython
+    ##################################### CYTHON ########################################
+    SET(CYTHON_SOURCES_DIR ${PYTHON_SOURCES_DIR}/cython)
+    ExternalProject_Add(cython
         ${EXTERNAL_PROJECT_LOG_ARGS}
         PREFIX                ${CYTHON_SOURCES_DIR}
         URL                   https://github.com/cython/cython/archive/0.25.2.tar.gz
@@ -130,19 +140,19 @@ ExternalProject_Add(cython
         INSTALL_COMMAND       ""
         BUILD_COMMAND         env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
         DEPENDS               python
-)
-####################################################################################
+    )
+    ####################################################################################
 
-##################################### NUMPY ########################################
-SET(NUMPY_SOURCES_DIR ${PYTHON_SOURCES_DIR}/numpy)
-SET(NUMPY_TAG_VERSION "v1.11.3")
-SET(NUMPY_VERSION "1.11.3")
+    ##################################### NUMPY ########################################
+    SET(NUMPY_SOURCES_DIR ${PYTHON_SOURCES_DIR}/numpy)
+    SET(NUMPY_TAG_VERSION "v1.11.3")
+    SET(NUMPY_VERSION "1.11.3")
 
-SET(EGG_NAME "")
-SET(PYTHON_NUMPY_INCLUDE_DIR "")
-IF(WIN32)
+    SET(EGG_NAME "")
+    SET(PYTHON_NUMPY_INCLUDE_DIR "")
+    IF(WIN32)
         SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}.egg")
-ELSE(WIN32)
+    ELSE(WIN32)
         IF(APPLE)
             SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}-${MACOS_VERSION}")
         ELSE(APPLE)
@@ -153,11 +163,9 @@ ELSE(WIN32)
         FOREACH(suffix x86_64 intel fat64 fat32 universal)
             LIST(APPEND PYTHON_NUMPY_INCLUDE_DIR ${PY_SITE_PACKAGES_PATH}/${EGG_NAME}-${suffix}.egg/numpy/core/include)
         ENDFOREACH()
-ENDIF(WIN32)
-
-INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR})
+    ENDIF(WIN32)
 
-ExternalProject_Add(numpy
+    ExternalProject_Add(numpy
         ${EXTERNAL_PROJECT_LOG_ARGS}
         GIT_REPOSITORY      https://github.com/numpy/numpy.git
         GIT_TAG             ${NUMPY_TAG_VERSION}
@@ -168,12 +176,12 @@ ExternalProject_Add(numpy
         INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
         BUILD_IN_SOURCE     1
         DEPENDS             python setuptools cython
-)
-####################################################################################
+    )
+    ####################################################################################
 
-##################################### WHEEL ########################################
-SET(WHEEL_SOURCES_DIR ${PYTHON_SOURCES_DIR}/wheel)
-ExternalProject_Add(wheel
+    ##################################### WHEEL ########################################
+    SET(WHEEL_SOURCES_DIR ${PYTHON_SOURCES_DIR}/wheel)
+    ExternalProject_Add(wheel
         ${EXTERNAL_PROJECT_LOG_ARGS}
         URL                 https://pypi.python.org/packages/source/w/wheel/wheel-0.29.0.tar.gz
         PREFIX              ${WHEEL_SOURCES_DIR}
@@ -183,12 +191,12 @@ ExternalProject_Add(wheel
         INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
         BUILD_IN_SOURCE     1
         DEPENDS             python setuptools
-)
-####################################################################################
+    )
+    ####################################################################################
 
-################################### PROTOBUF #######################################
-SET(PY_PROTOBUF_SOURCES_DIR ${PYTHON_SOURCES_DIR}/protobuf)
-ExternalProject_Add(python-protobuf
+    ################################### PROTOBUF #######################################
+    SET(PY_PROTOBUF_SOURCES_DIR ${PYTHON_SOURCES_DIR}/protobuf)
+    ExternalProject_Add(python-protobuf
         ${EXTERNAL_PROJECT_LOG_ARGS}
         URL                   https://pypi.python.org/packages/e0/b0/0a1b364fe8a7d177b4b7d4dca5b798500dc57a7273b93cca73931b305a6a/protobuf-3.1.0.post1.tar.gz
         URL_MD5               38b5fb160c768d2f8444d0c6d637ff91
@@ -196,9 +204,19 @@ ExternalProject_Add(python-protobuf
         BUILD_IN_SOURCE       1
         PATCH_COMMAND         ""
         CONFIGURE_COMMAND     ""
-    BUILD_COMMAND         env PATH=${PROTOBUF_INSTALL_DIR}/bin:$ENV{PATH} ${py_env} ${PYTHON_EXECUTABLE} setup.py build
-    INSTALL_COMMAND       env PATH=${PROTOBUF_INSTALL_DIR}/bin:$ENV{PATH} ${py_env} ${PYTHON_EXECUTABLE} setup.py install
+        BUILD_COMMAND         env ${py_env} ${PYTHON_EXECUTABLE} setup.py build
+        INSTALL_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
         DEPENDS               python setuptools six
-)
+    )
+    ####################################################################################
+
+    LIST(APPEND external_project_dependencies python setuptools six cython wheel python-protobuf numpy)
+
+ENDIF(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
+
+INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR})
+INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR})
 
-LIST(APPEND external_project_dependencies python setuptools six cython numpy wheel python-protobuf)
+MESSAGE("[Paddle] Python Executable: ${PYTHON_EXECUTABLE}")
+MESSAGE("[Paddle] Python Include: ${PYTHON_INCLUDE_DIRS}")
+MESSAGE("[Paddle] Python Libraries: ${PYTHON_LIBRARIES}")

paddle/api/CMakeLists.txt

@@ -94,6 +94,7 @@ add_dependencies(python_api_wheel python_swig_sources
   paddle_cuda)
 
 if(WITH_TESTING)
+    IF(NOT PY_PIP_FOUND)
         SET(PIP_SOURCES_DIR ${PYTHON_SOURCES_DIR}/pip)
         ExternalProject_Add(pip
             ${EXTERNAL_PROJECT_LOG_ARGS}
@@ -106,5 +107,6 @@ if(WITH_TESTING)
             BUILD_IN_SOURCE     1
             DEPENDS python setuptools python_api_wheel
         )
+    ENDIF()
     add_subdirectory(test)
 endif()

paddle/scripts/travis/before_install.osx.sh

 #!/bin/bash
 brew update
 brew tap homebrew/science
-brew install openblas md5sha1sum 
+brew install python
+sudo pip install --upgrade protobuf
+brew install swig openblas md5sha1sum protobuf

paddle/scripts/travis/build_and_test.sh

 #!/bin/bash
 source ./common.sh
 
-python -c 'import pip; print(pip.pep425tags.get_supported())'
-
-if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-  CMAKE_EXTRA="-DWITH_SWIG_PY=OFF"
-else
-  CMAKE_EXTRA="-DWITH_SWIG_PY=ON"
-fi
-
-cmake .. -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON -DON_COVERALLS=ON ${CMAKE_EXTRA}
-
 NPROC=1
 if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+  export PYTHONPATH=/opt/python/2.7.12/lib/python2.7/site-packages
+  export PYTHONHOME=/opt/python/2.7.12
+  export PATH=/opt/python/2.7.12/bin:${PATH}
+  cmake .. -DON_TRAVIS=ON -DON_COVERALLS=ON -DCOVERALLS_UPLOAD=ON
   NRPOC=`nproc`
   make -j $NPROC
   make coveralls
   sudo make install
 elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+  export PYTHONPATH=/usr/local/lib/python2.7/site-packages
+  cmake .. -DON_TRAVIS=ON -DON_COVERALLS=ON -DCOVERALLS_UPLOAD=ON -DWITH_SWIG_PY=ON
   NPROC=`sysctl -n hw.ncpu`
   make -j $NPROC
   env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j $NPROC"

paddle/utils/PythonUtil.cpp.in

@@ -195,8 +195,10 @@ extern const char enable_virtualenv_py[];
 }
 void initPython(int argc, char** argv) {
 #ifndef PADDLE_NO_PYTHON
-  char PythonHome[] = "@PYTHON_INSTALL_DIR@"; // NOLINT
-  Py_SetPythonHome(PythonHome);
+  char pyHome[] = "@PYTHON_INSTALL_DIR@"; // NOLINT
+  if (strlen(pyHome)) {
+    Py_SetPythonHome(pyHome);
+  }
   Py_SetProgramName(argv[0]);
   Py_Initialize();
   PySys_SetArgv(argc, argv);