Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into stat_usage

.travis.yml

@@ -21,13 +21,14 @@ addons:
     packages:
       - gcc-4.8
       - g++-4.8
-      - wget
       - git
       - build-essential
       - libatlas-base-dev
       - python
       - python-pip
       - python2.7-dev
+      - python-numpy
+      - python-wheel
       - curl
       - swig
       - graphviz
@@ -48,8 +49,7 @@ before_install:
     fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi
   - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi
-  - pip install --upgrade pip
-  - pip install wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker
+  - pip install numpy wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker
 script:
   - paddle/scripts/travis/main.sh
 notifications:

CMakeLists.txt

@@ -27,22 +27,23 @@ find_package(Threads REQUIRED)
 include(system)
 include(simd)
 
-###################### Configurations ############################
-option(WITH_DSO "Compile PaddlePaddle with dynamic linked libraries" ON)
-option(WITH_GPU "Compile PaddlePaddle with gpu" ${CUDA_FOUND})
-option(WITH_DOUBLE "Compile PaddlePaddle with double precision, otherwise use single precision" OFF)
-option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ${AVX_FOUND})
-option(WITH_PYTHON "Compile PaddlePaddle with python interpreter" ON)
-option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ON)
-option(WITH_RDMA "Compile PaddlePaddle with rdma support" OFF)
-option(WITH_TIMER "Compile PaddlePaddle use timer" OFF)
-option(WITH_PROFILER "Compile PaddlePaddle use gpu profiler" OFF)
-option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ON)
-option(WITH_DOC "Compile PaddlePaddle with documentation" OFF)
-option(WITH_SWIG_PY "Compile PaddlePaddle with py PaddlePaddle prediction api" ON)
-option(ON_TRAVIS "Running test on travis-ci or not." OFF)
-option(ON_COVERALLS "Generating code coverage data on coveralls or not." OFF)
-option(COVERALLS_UPLOAD "Uploading the generated coveralls json." ON)
+################################ Configurations #######################################
+option(WITH_GPU         "Compile PaddlePaddle with NVIDIA GPU"          ${CUDA_FOUND})
+option(WITH_AVX         "Compile PaddlePaddle with AVX intrinsics"      ${AVX_FOUND})
+option(WITH_DSO         "Compile PaddlePaddle with dynamic linked CUDA" OFF)
+option(WITH_TESTING     "Compile PaddlePaddle with unit testing"        ON)
+option(WITH_SWIG_PY     "Compile PaddlePaddle with inference api"       ON)
+option(WITH_STYLE_CHECK "Compile PaddlePaddle with style check"         ON)
+option(WITH_PYTHON      "Compile PaddlePaddle with python interpreter"  ON)
+option(WITH_DOUBLE      "Compile PaddlePaddle with double precision"    OFF)
+option(WITH_RDMA        "Compile PaddlePaddle with RDMA support"        OFF)
+option(WITH_TIMER       "Compile PaddlePaddle with stats timer"         OFF)
+option(WITH_PROFILER    "Compile PaddlePaddle with GPU profiler"        OFF)
+option(WITH_DOC         "Compile PaddlePaddle with documentation"       OFF)
+option(ON_COVERALLS     "Compile PaddlePaddle with code coverage"       OFF)
+option(COVERALLS_UPLOAD "Package code coverage data to coveralls"       OFF)
+option(ON_TRAVIS        "Exclude special unit test on Travis CI"        OFF)
+########################################################################################
 
 include(external/zlib)      # download, build, install zlib
 include(external/gflags)    # download, build, install gflags
@@ -63,7 +64,6 @@ include(flags)              # set paddle compile flags
 include(cudnn)              # set cudnn libraries
 include(version)            # set PADDLE_VERSION
 include(coveralls)          # set code coverage
-include(python_module)      # set python module
 
 include(configure)          # add paddle env configuration
 

cmake/configure.cmake

@@ -12,6 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+if(NOT WITH_PYTHON)
+    add_definitions(-DPADDLE_NO_PYTHON)
+endif(NOT WITH_PYTHON)
+
 if(WITH_DSO)
     add_definitions(-DPADDLE_USE_DSO)
 endif(WITH_DSO)

cmake/external/openblas.cmake

@@ -15,6 +15,7 @@
 INCLUDE(cblas)
 
 IF(NOT ${CBLAS_FOUND})
+    MESSAGE(FATAL_ERROR "Please install OpenBlas, MKL or ATLAS.")
     INCLUDE(ExternalProject)
 
     SET(CBLAS_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/openblas)

cmake/external/python.cmake

@@ -13,192 +13,210 @@
 # limitations under the License.
 
 INCLUDE(ExternalProject)
+INCLUDE(python_module)
+
+FIND_PACKAGE(PythonInterp 2.7)
+FIND_PACKAGE(PythonLibs 2.7)
+
+SET(py_env "")
+
+IF(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
+    find_python_module(pip REQUIRED)
+    find_python_module(numpy REQUIRED)
+    find_python_module(wheel REQUIRED)
+    find_python_module(google.protobuf REQUIRED)
+    FIND_PACKAGE(NumPy REQUIRED)
+ELSE(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
+    ##################################### PYTHON ########################################
+    SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/python)
+    SET(PYTHON_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/python)
+    SET(_python_DIR ${PYTHON_INSTALL_DIR})
+
+    IF(UNIX)
+        SET(PYTHON_FOUND ON)
+        SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include/python2.7" CACHE PATH "Python include dir" FORCE)
+        SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/lib/libpython2.7.a" CACHE FILEPATH "Python library" FORCE)
+        SET(PYTHON_EXECUTABLE ${PYTHON_INSTALL_DIR}/bin/python CACHE FILEPATH "Python executable" FORCE)
+        SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/lib/python2.7/site-packages" CACHE PATH "Python site-packages path" FORCE)
+    ELSEIF(WIN32)
+        SET(PYTHON_FOUND ON)
+        SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include" CACHE PATH "Python include dir" FORCE)
+        SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/libs/python27.lib" CACHE FILEPATH "Python library" FORCE)
+        SET(PYTHON_EXECUTABLE "${PYTHON_INSTALL_DIR}/bin/python.exe" CACHE FILEPATH "Python executable" FORCE)
+        SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/Lib/site-packages" CACHE PATH "Python site-packages path" FORCE)
+    ELSE()
+        MESSAGE(FATAL_ERROR "Unknown system !")
+    ENDIF()
 
-
-##################################### PYTHON ########################################
-SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/python)
-SET(PYTHON_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/python)
-SET(_python_DIR ${PYTHON_INSTALL_DIR})
-
-IF(UNIX)
-    SET(PYTHON_FOUND ON)
-    SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include/python2.7" CACHE PATH "Python include dir" FORCE)
-    SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/lib/libpython2.7.a" CACHE FILEPATH "Python library" FORCE)
-    SET(PYTHON_EXECUTABLE ${PYTHON_INSTALL_DIR}/bin/python CACHE FILEPATH "Python executable" FORCE)
-    SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/lib/python2.7/site-packages" CACHE PATH "Python site-packages path" FORCE)
-ELSEIF(WIN32)
-    SET(PYTHON_FOUND ON)
-    SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include" CACHE PATH "Python include dir" FORCE)
-    SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/libs/python27.lib" CACHE FILEPATH "Python library" FORCE)
-    SET(PYTHON_EXECUTABLE "${PYTHON_INSTALL_DIR}/bin/python.exe" CACHE FILEPATH "Python executable" FORCE)
-    SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/Lib/site-packages" CACHE PATH "Python site-packages path" FORCE)
-ELSE()
-    MESSAGE(FATAL_ERROR "Unknown system !")
-ENDIF()
-
-SET(py_env
-    PATH=${PYTHON_INSTALL_DIR}/bin/:$ENV{PATH}
-    PYTHONHOME=${PYTHON_INSTALL_DIR}
-    PYTHONPATH=${PYTHON_INSTALL_DIR}/lib:${PYTHON_INSTALL_DIR}/lib/python2.7:${PY_SITE_PACKAGES_PATH})
-
-INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR})
-
-IF(APPLE)
-    LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS
-        -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON
-        )
-ENDIF()
-
-SET(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS)
-
-# Force Python build to "Release".
-IF(CMAKE_CONFIGURATION_TYPES)
-    SET(SAVED_CMAKE_CFG_INTDIR ${CMAKE_CFG_INTDIR})
-    SET(CMAKE_CFG_INTDIR "Release")
-ELSE()
-    LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS
-        -DCMAKE_BUILD_TYPE:STRING=Release
-        )
-ENDIF()
-
-ExternalProject_Add(python
-    ${EXTERNAL_PROJECT_LOG_ARGS}
-    GIT_REPOSITORY    "https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git"
-    PREFIX            ${PYTHON_SOURCES_DIR}
-    UPDATE_COMMAND    ""
-    CMAKE_ARGS        -DPYTHON_VERSION=2.7.12
-    CMAKE_ARGS        -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-    CMAKE_ARGS        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-    CMAKE_CACHE_ARGS
-        -DCMAKE_INSTALL_PREFIX:PATH=${PYTHON_INSTALL_DIR}
-        -DBUILD_LIBPYTHON_SHARED:BOOL=OFF
-        -DUSE_SYSTEM_LIBRARIES:BOOL=OFF
-        -DZLIB_ROOT:FILEPATH=${ZLIB_ROOT}
-        -DZLIB_INCLUDE_DIR:PATH=${ZLIB_INCLUDE_DIR}
-        -DZLIB_LIBRARY:FILEPATH=${ZLIB_LIBRARIES}
-        -DDOWNLOAD_SOURCES:BOOL=ON
-        -DINSTALL_WINDOWS_TRADITIONAL:BOOL=OFF
-        ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS}
-        ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS}
-    DEPENDS zlib
-)
-####################################################################################
-
-##################################### SETUPTOOLS ###################################
-SET(SETUPTOOLS_SOURCES_DIR ${PYTHON_SOURCES_DIR}/setuptools)
-ExternalProject_Add(setuptools
-    ${EXTERNAL_PROJECT_LOG_ARGS}
-    PREFIX              ${SETUPTOOLS_SOURCES_DIR}
-    URL                 "https://pypi.python.org/packages/source/s/setuptools/setuptools-18.3.2.tar.gz"
-    BUILD_IN_SOURCE     1
-    PATCH_COMMAND       ""
-    UPDATE_COMMAND      ""
-    CONFIGURE_COMMAND   ""
-    INSTALL_COMMAND     ""
-    BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-    DEPENDS             python zlib
-)
-#####################################################################################
-
-##################################### SIX ###########################################
-SET(SIX_SOURCES_DIR ${PYTHON_SOURCES_DIR}/six)
-ExternalProject_Add(six
-    ${EXTERNAL_PROJECT_LOG_ARGS}
-    PREFIX              ${SIX_SOURCES_DIR}
-    URL                 https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz
-    BUILD_IN_SOURCE     1
-    PATCH_COMMAND       ""
-    UPDATE_COMMAND      ""
-    CONFIGURE_COMMAND   ""
-    INSTALL_COMMAND     ""
-    BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-    DEPENDS             python setuptools
-)
-#####################################################################################
-
-##################################### CYTHON ########################################
-SET(CYTHON_SOURCES_DIR ${PYTHON_SOURCES_DIR}/cython)
-ExternalProject_Add(cython
-    ${EXTERNAL_PROJECT_LOG_ARGS}
-    PREFIX                ${CYTHON_SOURCES_DIR}
-    URL                   https://github.com/cython/cython/archive/0.25.2.tar.gz
-    GIT_TAG               0.25.2
-    BUILD_IN_SOURCE       1
-    CONFIGURE_COMMAND     ""
-    PATCH_COMMAND         ""
-    UPDATE_COMMAND        ""
-    INSTALL_COMMAND       ""
-    BUILD_COMMAND         env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-    DEPENDS               python
-)
-####################################################################################
-
-##################################### NUMPY ########################################
-SET(NUMPY_SOURCES_DIR ${PYTHON_SOURCES_DIR}/numpy)
-SET(NUMPY_TAG_VERSION "v1.11.3")
-SET(NUMPY_VERSION "1.11.3")
-
-SET(EGG_NAME "")
-SET(PYTHON_NUMPY_INCLUDE_DIR "")
-IF(WIN32)
-    SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}.egg")
-ELSE(WIN32)
     IF(APPLE)
-        SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}-${MACOS_VERSION}")
-    ELSE(APPLE)
-        SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-linux")
-        SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-linux")
-    ENDIF(APPLE)
-
-    FOREACH(suffix x86_64 intel fat64 fat32 universal)
-        LIST(APPEND PYTHON_NUMPY_INCLUDE_DIR ${PY_SITE_PACKAGES_PATH}/${EGG_NAME}-${suffix}.egg/numpy/core/include)
-    ENDFOREACH()
-ENDIF(WIN32)
+        LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS
+            -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON
+            )
+    ENDIF()
+
+    SET(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS)
+
+    # Force Python build to "Release".
+    IF(CMAKE_CONFIGURATION_TYPES)
+        SET(SAVED_CMAKE_CFG_INTDIR ${CMAKE_CFG_INTDIR})
+        SET(CMAKE_CFG_INTDIR "Release")
+    ELSE()
+        LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS
+            -DCMAKE_BUILD_TYPE:STRING=Release
+            )
+    ENDIF()
+
+    ExternalProject_Add(python
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        GIT_REPOSITORY    "https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git"
+        PREFIX            ${PYTHON_SOURCES_DIR}
+        UPDATE_COMMAND    ""
+        CMAKE_ARGS        -DPYTHON_VERSION=2.7.12
+        CMAKE_ARGS        -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+        CMAKE_ARGS        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+        CMAKE_CACHE_ARGS
+            -DCMAKE_INSTALL_PREFIX:PATH=${PYTHON_INSTALL_DIR}
+            -DBUILD_LIBPYTHON_SHARED:BOOL=OFF
+            -DUSE_SYSTEM_LIBRARIES:BOOL=OFF
+            -DZLIB_ROOT:FILEPATH=${ZLIB_ROOT}
+            -DZLIB_INCLUDE_DIR:PATH=${ZLIB_INCLUDE_DIR}
+            -DZLIB_LIBRARY:FILEPATH=${ZLIB_LIBRARIES}
+            -DDOWNLOAD_SOURCES:BOOL=ON
+            -DINSTALL_WINDOWS_TRADITIONAL:BOOL=OFF
+            ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS}
+            ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS}
+        DEPENDS zlib
+    )
+
+    SET(py_env
+        PATH=${PYTHON_INSTALL_DIR}/bin
+        PYTHONHOME=${PYTHON_INSTALL_DIR}
+        PYTHONPATH=${PYTHON_INSTALL_DIR}/lib:${PYTHON_INSTALL_DIR}/lib/python2.7:${PY_SITE_PACKAGES_PATH})
+    ####################################################################################
+
+    ##################################### SETUPTOOLS ###################################
+    SET(SETUPTOOLS_SOURCES_DIR ${PYTHON_SOURCES_DIR}/setuptools)
+    ExternalProject_Add(setuptools
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        PREFIX              ${SETUPTOOLS_SOURCES_DIR}
+        URL                 "https://pypi.python.org/packages/source/s/setuptools/setuptools-18.3.2.tar.gz"
+        BUILD_IN_SOURCE     1
+        PATCH_COMMAND       ""
+        UPDATE_COMMAND      ""
+        CONFIGURE_COMMAND   ""
+        INSTALL_COMMAND     ""
+        BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
+        DEPENDS             python zlib
+    )
+    #####################################################################################
+
+    ##################################### SIX ###########################################
+    SET(SIX_SOURCES_DIR ${PYTHON_SOURCES_DIR}/six)
+    ExternalProject_Add(six
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        PREFIX              ${SIX_SOURCES_DIR}
+        URL                 https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz
+        BUILD_IN_SOURCE     1
+        PATCH_COMMAND       ""
+        UPDATE_COMMAND      ""
+        CONFIGURE_COMMAND   ""
+        INSTALL_COMMAND     ""
+        BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
+        DEPENDS             python setuptools
+    )
+    #####################################################################################
+
+    ##################################### CYTHON ########################################
+    SET(CYTHON_SOURCES_DIR ${PYTHON_SOURCES_DIR}/cython)
+    ExternalProject_Add(cython
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        PREFIX                ${CYTHON_SOURCES_DIR}
+        URL                   https://github.com/cython/cython/archive/0.25.2.tar.gz
+        GIT_TAG               0.25.2
+        BUILD_IN_SOURCE       1
+        CONFIGURE_COMMAND     ""
+        PATCH_COMMAND         ""
+        UPDATE_COMMAND        ""
+        INSTALL_COMMAND       ""
+        BUILD_COMMAND         env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
+        DEPENDS               python
+    )
+    ####################################################################################
+
+    ##################################### NUMPY ########################################
+    SET(NUMPY_SOURCES_DIR ${PYTHON_SOURCES_DIR}/numpy)
+    SET(NUMPY_TAG_VERSION "v1.11.3")
+    SET(NUMPY_VERSION "1.11.3")
+
+    SET(EGG_NAME "")
+    SET(PYTHON_NUMPY_INCLUDE_DIR "")
+    IF(WIN32)
+        SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}.egg")
+    ELSE(WIN32)
+        IF(APPLE)
+            SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}-${MACOS_VERSION}")
+        ELSE(APPLE)
+            SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-linux")
+            SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-linux")
+        ENDIF(APPLE)
+
+        FOREACH(suffix x86_64 intel fat64 fat32 universal)
+            LIST(APPEND PYTHON_NUMPY_INCLUDE_DIR ${PY_SITE_PACKAGES_PATH}/${EGG_NAME}-${suffix}.egg/numpy/core/include)
+        ENDFOREACH()
+    ENDIF(WIN32)
+
+    ExternalProject_Add(numpy
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        GIT_REPOSITORY      https://github.com/numpy/numpy.git
+        GIT_TAG             ${NUMPY_TAG_VERSION}
+        CONFIGURE_COMMAND   ""
+        UPDATE_COMMAND      ""
+        PREFIX              ${NUMPY_SOURCES_DIR}
+        BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py build
+        INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
+        BUILD_IN_SOURCE     1
+        DEPENDS             python setuptools cython
+    )
+    ####################################################################################
+
+    ##################################### WHEEL ########################################
+    SET(WHEEL_SOURCES_DIR ${PYTHON_SOURCES_DIR}/wheel)
+    ExternalProject_Add(wheel
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        URL                 https://pypi.python.org/packages/source/w/wheel/wheel-0.29.0.tar.gz
+        PREFIX              ${WHEEL_SOURCES_DIR}
+        CONFIGURE_COMMAND   ""
+        UPDATE_COMMAND      ""
+        BUILD_COMMAND       ""
+        INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
+        BUILD_IN_SOURCE     1
+        DEPENDS             python setuptools
+    )
+    ####################################################################################
+
+    ################################### PROTOBUF #######################################
+    SET(PY_PROTOBUF_SOURCES_DIR ${PYTHON_SOURCES_DIR}/protobuf)
+    ExternalProject_Add(python-protobuf
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        URL                   https://pypi.python.org/packages/e0/b0/0a1b364fe8a7d177b4b7d4dca5b798500dc57a7273b93cca73931b305a6a/protobuf-3.1.0.post1.tar.gz
+        URL_MD5               38b5fb160c768d2f8444d0c6d637ff91
+        PREFIX                ${PY_PROTOBUF_SOURCES_DIR}
+        BUILD_IN_SOURCE       1
+        PATCH_COMMAND         ""
+        CONFIGURE_COMMAND     ""
+        BUILD_COMMAND         env ${py_env} ${PYTHON_EXECUTABLE} setup.py build
+        INSTALL_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
+        DEPENDS               python setuptools six
+    )
+    ####################################################################################
+
+    LIST(APPEND external_project_dependencies python setuptools six cython wheel python-protobuf numpy)
+
+ENDIF(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
 
+INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR})
 INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR})
 
-ExternalProject_Add(numpy
-    ${EXTERNAL_PROJECT_LOG_ARGS}
-    GIT_REPOSITORY      https://github.com/numpy/numpy.git
-    GIT_TAG             ${NUMPY_TAG_VERSION}
-    CONFIGURE_COMMAND   ""
-    UPDATE_COMMAND      ""
-    PREFIX              ${NUMPY_SOURCES_DIR}
-    BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py build
-    INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-    BUILD_IN_SOURCE     1
-    DEPENDS             python setuptools cython
-)
-####################################################################################
-
-##################################### WHEEL ########################################
-SET(WHEEL_SOURCES_DIR ${PYTHON_SOURCES_DIR}/wheel)
-ExternalProject_Add(wheel
-    ${EXTERNAL_PROJECT_LOG_ARGS}
-    URL                 https://pypi.python.org/packages/source/w/wheel/wheel-0.29.0.tar.gz
-    PREFIX              ${WHEEL_SOURCES_DIR}
-    CONFIGURE_COMMAND   ""
-    UPDATE_COMMAND      ""
-    BUILD_COMMAND       ""
-    INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-    BUILD_IN_SOURCE     1
-    DEPENDS             python setuptools
-)
-####################################################################################
-
-################################### PROTOBUF #######################################
-SET(PY_PROTOBUF_SOURCES_DIR ${PYTHON_SOURCES_DIR}/protobuf)
-ExternalProject_Add(python-protobuf
-    ${EXTERNAL_PROJECT_LOG_ARGS}
-    URL                   https://pypi.python.org/packages/e0/b0/0a1b364fe8a7d177b4b7d4dca5b798500dc57a7273b93cca73931b305a6a/protobuf-3.1.0.post1.tar.gz
-    URL_MD5               38b5fb160c768d2f8444d0c6d637ff91
-    PREFIX                ${PY_PROTOBUF_SOURCES_DIR}
-    BUILD_IN_SOURCE       1
-    PATCH_COMMAND         ""
-    CONFIGURE_COMMAND     ""
-    BUILD_COMMAND         env PATH=${PROTOBUF_INSTALL_DIR}/bin:$ENV{PATH} ${py_env} ${PYTHON_EXECUTABLE} setup.py build
-    INSTALL_COMMAND       env PATH=${PROTOBUF_INSTALL_DIR}/bin:$ENV{PATH} ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-    DEPENDS               python setuptools six
-)
-
-LIST(APPEND external_project_dependencies python setuptools six cython numpy wheel python-protobuf)
+MESSAGE("[Paddle] Python Executable: ${PYTHON_EXECUTABLE}")
+MESSAGE("[Paddle] Python Include: ${PYTHON_INCLUDE_DIRS}")
+MESSAGE("[Paddle] Python Libraries: ${PYTHON_LIBRARIES}")

cmake/external/warpctc.cmake

@@ -53,6 +53,8 @@ ExternalProject_Add(
     CMAKE_ARGS      -DCMAKE_INSTALL_PREFIX=${WARPCTC_INSTALL_DIR}
     CMAKE_ARGS      -DWITH_GPU=${WITH_GPU}
     CMAKE_ARGS      -DWITH_OMP=${USE_OMP}
+    CMAKE_ARGS      -DWITH_TORCH=OFF
+    CMAKE_ARGS      -DBUILD_SHARED=ON
 )
 
 LIST(APPEND external_project_dependencies warpctc)

cmake/util.cmake

@@ -120,6 +120,7 @@ function(link_paddle_exe TARGET_NAME)
             target_link_libraries(${TARGET_NAME} rt)
         endif()
     endif()
+
     add_dependencies(${TARGET_NAME} ${external_project_dependencies})
 endfunction()
 

demo/traffic_prediction/README

+run by:
+cd ./data
+sh get_data.sh
+cd ..
+sh train.sh
+sh predict.sh
+

demo/traffic_prediction/data/get_data.sh

+#!/bin/bash
+# Copyright (c) 2016 PaddlePaddle Authors, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -x
+
+DIR="$( cd "$(dirname "$0")" ; pwd -P )"
+cd $DIR
+
+#download the dataset
+echo "Downloading traffic data..."
+wget http://paddlepaddle.cdn.bcebos.com/demo/traffic/traffic_data.tar.gz
+
+#extract package
+echo "Unzipping..."
+tar -zxvf traffic_data.tar.gz
+
+echo "data/speeds.csv" > train.list
+echo "data/speeds.csv" > test.list
+echo "data/speeds.csv" > pred.list
+
+echo "Done."

demo/traffic_prediction/dataprovider.py

+# Copyright (c) 2016 PaddlePaddle Authors, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle.trainer.PyDataProvider2 import *
+import sys
+import numpy as np
+TERM_NUM = 24
+FORECASTING_NUM = 24
+LABEL_VALUE_NUM = 4
+
+
+def initHook(settings, file_list, **kwargs):
+    """
+    Init hook is invoked before process data. It will set obj.slots and store data meta.
+
+    :param settings: global object. It will passed to process routine.
+    :type obj: object
+    :param file_list: the meta file object, which passed from trainer_config.py,but unused in this function.
+    :param kwargs: unused other arguments.
+    """
+    del kwargs  #unused 
+
+    settings.pool_size = sys.maxint
+    #Use a time seires of the past as feature.
+    #Dense_vector's expression form is [float,float,...,float]
+    settings.input_types = [dense_vector(TERM_NUM)]
+    #There are next FORECASTING_NUM fragments you need predict.
+    #Every predicted condition at time point has four states.
+    for i in range(FORECASTING_NUM):
+        settings.input_types.append(integer_value(LABEL_VALUE_NUM))
+
+
+@provider(
+    init_hook=initHook, cache=CacheType.CACHE_PASS_IN_MEM, should_shuffle=True)
+def process(settings, file_name):
+    with open(file_name) as f:
+        #abandon fields name
+        f.next()
+        for row_num, line in enumerate(f):
+            speeds = map(int, line.rstrip('\r\n').split(",")[1:])
+            # Get the max index.
+            end_time = len(speeds)
+            # Scanning and generating samples
+            for i in range(TERM_NUM, end_time - FORECASTING_NUM):
+                # For dense slot
+                pre_spd = map(float, speeds[i - TERM_NUM:i])
+
+                # Integer value need predicting, values start from 0, so every one minus 1.
+                fol_spd = [j - 1 for j in speeds[i:i + FORECASTING_NUM]]
+
+                # Predicting label is missing, abandon the sample.
+                if -1 in fol_spd:
+                    continue
+                yield [pre_spd] + fol_spd
+
+
+def predict_initHook(settings, file_list, **kwargs):
+    settings.pool_size = sys.maxint
+    settings.input_types = [dense_vector(TERM_NUM)]
+
+
+@provider(init_hook=predict_initHook, should_shuffle=False)
+def process_predict(settings, file_name):
+    with open(file_name) as f:
+        #abandon fields name
+        f.next()
+        for row_num, line in enumerate(f):
+            speeds = map(int, line.rstrip('\r\n').split(","))
+            end_time = len(speeds)
+            pre_spd = map(float, speeds[end_time - TERM_NUM:end_time])
+            yield pre_spd

demo/traffic_prediction/gen_result.py

+# Copyright (c) 2016 PaddlePaddle Authors, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+res = []
+with open('./rank-00000') as f:
+    for line in f:
+        pred = map(int, line.strip('\r\n;').split(";"))
+        #raw prediction range from 0 to 3
+        res.append([i + 1 for i in pred])
+
+file_name = open('./data/pred.list').read().strip('\r\n')
+
+FORECASTING_NUM = 24
+header = [
+    'id',
+    '201604200805',
+    '201604200810',
+    '201604200815',
+    '201604200820',
+    '201604200825',
+    '201604200830',
+    '201604200835',
+    '201604200840',
+    '201604200845',
+    '201604200850',
+    '201604200855',
+    '201604200900',
+    '201604200905',
+    '201604200910',
+    '201604200915',
+    '201604200920',
+    '201604200925',
+    '201604200930',
+    '201604200935',
+    '201604200940',
+    '201604200945',
+    '201604200950',
+    '201604200955',
+    '201604201000',
+]
+###################
+## To CSV format ##
+###################
+with open(file_name) as f:
+    f.next()
+    print ','.join(header)
+    for row_num, line in enumerate(f):
+        fields = line.rstrip('\r\n').split(',')
+        linkid = fields[0]
+        print linkid + ',' + ','.join(map(str, res[row_num]))

demo/traffic_prediction/predict.sh

+#!/bin/bash
+# Copyright (c) 2016 PaddlePaddle Authors, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+
+cfg=trainer_config.py
+# pass choice 
+model="output/pass-00000"
+paddle train \
+    --config=$cfg \
+    --use_gpu=false \
+    --job=test \
+    --init_model_path=$model \
+    --config_args=is_predict=1 \
+    --predict_output_dir=. 
+
+python gen_result.py > result.txt
+
+rm -rf rank-00000

demo/traffic_prediction/train.sh

+#!/bin/bash
+# Copyright (c) 2016 PaddlePaddle Authors, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+
+cfg=trainer_config.py
+paddle train \
+  --config=$cfg \
+  --save_dir=./output \
+  --trainer_count=4 \
+  --log_period=1000 \
+  --dot_period=10 \
+  --num_passes=10 \
+  --use_gpu=false \
+  --show_parameter_stats_period=3000 \
+  2>&1 | tee 'train.log'

demo/traffic_prediction/trainer_config.py

+# Copyright (c) 2016 PaddlePaddle Authors, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from paddle.trainer_config_helpers import *
+
+################################### DATA Configuration #############################################
+is_predict = get_config_arg('is_predict', bool, False)
+trn = './data/train.list' if not is_predict else None
+tst = './data/test.list' if not is_predict else './data/pred.list'
+process = 'process' if not is_predict else 'process_predict'
+define_py_data_sources2(
+    train_list=trn, test_list=tst, module="dataprovider", obj=process)
+################################### Parameter Configuaration #######################################
+TERM_NUM = 24
+FORECASTING_NUM = 24
+emb_size = 16
+batch_size = 128 if not is_predict else 1
+settings(
+    batch_size=batch_size,
+    learning_rate=1e-3,
+    learning_method=RMSPropOptimizer())
+################################### Algorithm Configuration ########################################
+
+output_label = []
+
+link_encode = data_layer(name='link_encode', size=TERM_NUM)
+for i in xrange(FORECASTING_NUM):
+    # Each task share same weight.
+    link_param = ParamAttr(
+        name='_link_vec.w', initial_max=1.0, initial_min=-1.0)
+    link_vec = fc_layer(input=link_encode, size=emb_size, param_attr=link_param)
+    score = fc_layer(input=link_vec, size=4, act=SoftmaxActivation())
+    if is_predict:
+        maxid = maxid_layer(score)
+        output_label.append(maxid)
+    else:
+        # Multi-task training.
+        label = data_layer(name='label_%dmin' % ((i + 1) * 5), size=4)
+        cls = classification_cost(
+            input=score, name="cost_%dmin" % ((i + 1) * 5), label=label)
+        output_label.append(cls)
+outputs(output_label)

paddle/api/CMakeLists.txt

@@ -94,17 +94,19 @@ add_dependencies(python_api_wheel python_swig_sources
   paddle_cuda)
 
 if(WITH_TESTING)
-    SET(PIP_SOURCES_DIR ${PYTHON_SOURCES_DIR}/pip)
-    ExternalProject_Add(pip
-        ${EXTERNAL_PROJECT_LOG_ARGS}
-        GIT_REPOSITORY      https://github.com/pypa/pip.git
-        GIT_TAG             9.0.1
-        PREFIX              ${PIP_SOURCES_DIR}
-        CONFIGURE_COMMAND   ""
-        BUILD_COMMAND       ""
-        INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-        BUILD_IN_SOURCE     1
-        DEPENDS python setuptools python_api_wheel
-    )
+    IF(NOT PY_PIP_FOUND)
+        SET(PIP_SOURCES_DIR ${PYTHON_SOURCES_DIR}/pip)
+        ExternalProject_Add(pip
+            ${EXTERNAL_PROJECT_LOG_ARGS}
+            GIT_REPOSITORY      https://github.com/pypa/pip.git
+            GIT_TAG             9.0.1
+            PREFIX              ${PIP_SOURCES_DIR}
+            CONFIGURE_COMMAND   ""
+            BUILD_COMMAND       ""
+            INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
+            BUILD_IN_SOURCE     1
+            DEPENDS python setuptools python_api_wheel
+        )
+    ENDIF()
     add_subdirectory(test)
 endif()

paddle/cuda/src/hl_warpctc_wrap.cc

@@ -29,7 +29,6 @@ void* warpctc_dso_handle = nullptr;
  * false, you need to add the path of libwarp-ctc.so to
  * the linked-libs of paddle or to LD_PRELOAD.
  */
-#ifdef PADDLE_USE_DSO
 #define DYNAMIC_LOAD_WARPCTC_WRAP(__name)                              \
   struct DynLoad__##__name {                                           \
     template <typename... Args>                                        \
@@ -41,15 +40,6 @@ void* warpctc_dso_handle = nullptr;
       return reinterpret_cast<warpctcFunc>(p_##_name)(args...);        \
     }                                                                  \
   } __name;  // struct DynLoad__##__name
-#else
-#define DYNAMIC_LOAD_WARPCTC_WRAP(__name)                        \
-  struct DynLoad__##__name {                                     \
-    template <typename... Args>                                  \
-    auto operator()(Args... args) -> decltype(__name(args...)) { \
-      return __name(args...);                                    \
-    }                                                            \
-  } __name;  // struct DynLoad__##__name
-#endif
 
 // include all needed warp-ctc functions
 DYNAMIC_LOAD_WARPCTC_WRAP(get_warpctc_version)

paddle/scripts/travis/before_install.osx.sh

 #!/bin/bash
 brew update
 brew tap homebrew/science
-brew install openblas md5sha1sum 
+brew install python
+sudo pip install --upgrade protobuf
+brew install swig openblas md5sha1sum protobuf

paddle/scripts/travis/build_and_test.sh

 #!/bin/bash
 source ./common.sh
 
-python -c 'import pip; print(pip.pep425tags.get_supported())'
-
-if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-  CMAKE_EXTRA="-DWITH_SWIG_PY=OFF"
-else
-  CMAKE_EXTRA="-DWITH_SWIG_PY=ON"
-fi
-
-cmake .. -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON -DON_COVERALLS=ON ${CMAKE_EXTRA}
-
 NPROC=1
 if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+  export PYTHONPATH=/opt/python/2.7.12/lib/python2.7/site-packages
+  export PYTHONHOME=/opt/python/2.7.12
+  export PATH=/opt/python/2.7.12/bin:${PATH}
+  cmake .. -DON_TRAVIS=ON -DON_COVERALLS=ON -DCOVERALLS_UPLOAD=ON
   NRPOC=`nproc`
   make -j $NPROC
   make coveralls
   sudo make install
 elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+  export PYTHONPATH=/usr/local/lib/python2.7/site-packages
+  cmake .. -DON_TRAVIS=ON -DON_COVERALLS=ON -DCOVERALLS_UPLOAD=ON
   NPROC=`sysctl -n hw.ncpu`
   make -j $NPROC
-  env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j $NPROC"
-  sudo make install
-  sudo paddle version
 fi

paddle/utils/PythonUtil.cpp.in

@@ -195,8 +195,10 @@ extern const char enable_virtualenv_py[];
 }
 void initPython(int argc, char** argv) {
 #ifndef PADDLE_NO_PYTHON
-  char PythonHome[] = "@PYTHON_INSTALL_DIR@"; // NOLINT
-  Py_SetPythonHome(PythonHome);
+  char pyHome[] = "@PYTHON_INSTALL_DIR@"; // NOLINT
+  if (strlen(pyHome)) {
+    Py_SetPythonHome(pyHome);
+  }
   Py_SetProgramName(argv[0]);
   Py_Initialize();
   PySys_SetArgv(argc, argv);