From aee0857838ee41f46237f0b6561242d523116a0c Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 28 Dec 2016 13:49:01 +0800 Subject: [PATCH] Clean Travis ci and fix bug --- .gitmodules | 3 --- .pre-commit-config.yaml | 4 ++-- .travis.yml | 6 ----- CMakeLists.txt | 1 - cmake/external/gflags.cmake | 1 + cmake/external/glog.cmake | 1 + cmake/external/gtest.cmake | 1 + cmake/external/protobuf.cmake | 23 +++++++++++-------- cmake/external/warpctc.cmake | 1 + paddle/api/paddle_api_config.py.in | 10 ++++---- paddle/api/paddle_ld_flags.py | 6 ++--- paddle/cuda/CMakeLists.txt | 2 +- paddle/gserver/tests/CMakeLists.txt | 2 +- paddle/scripts/travis/before_install.linux.sh | 18 --------------- paddle/scripts/travis/before_install.osx.sh | 8 +------ paddle/scripts/travis/build_submodules.sh | 20 ---------------- 16 files changed, 30 insertions(+), 77 deletions(-) delete mode 100644 .gitmodules delete mode 100755 paddle/scripts/travis/before_install.linux.sh delete mode 100755 paddle/scripts/travis/build_submodules.sh diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index f635e65784a..00000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "warp-ctc"] - path = warp-ctc - url = https://github.com/baidu-research/warp-ctc.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b9902a863d8..a6e45028ebc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ sha: c25201a00e6b0514370501050cf2a8538ac12270 hooks: - id: remove-crlf - files: (?!.*warp-ctc)^.*$ + files: (?!.*third_party)^.*$ - repo: https://github.com/reyoung/mirrors-yapf.git sha: v0.13.2 hooks: @@ -15,7 +15,7 @@ - id: check-merge-conflict - id: check-symlinks - id: detect-private-key - files: (?!.*warp-ctc)^.*$ + files: (?!.*third_party)^.*$ - id: end-of-file-fixer - repo: https://github.com/PaddlePaddle/clang-format-pre-commit-hook.git sha: 28c0ea8a67a3e2dbbf4822ef44e85b63a0080a29 diff --git a/.travis.yml b/.travis.yml index 047ca6ffe79..b49d4638d71 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,16 +24,11 @@ addons: - wget - git - build-essential - - libatlas-base-dev - python - python-pip - python2.7-dev - - m4 - python-numpy - python-wheel - - libgoogle-glog-dev - - libgflags-dev - - libgtest-dev - curl - lcov - graphviz @@ -53,7 +48,6 @@ before_install: fi fi fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi - pip install wheel protobuf sphinx recommonmark virtualenv numpy sphinx_rtd_theme pre-commit requests==2.9.2 LinkChecker diff --git a/CMakeLists.txt b/CMakeLists.txt index 7db4c9f1d1f..784876f0899 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,7 +65,6 @@ set(EXTERNAL_LIBS ${GLOG_LIBRARIES} ${CBLAS_LIBRARIES} ${PROTOBUF_LIBRARIES} - ${WARPCTC_LIBRARIES} ${ZLIB_LIBRARIES} ) diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake index 128d50cec85..55f9a4c3e6b 100644 --- a/cmake/external/gflags.cmake +++ b/cmake/external/gflags.cmake @@ -22,6 +22,7 @@ ExternalProject_Add( GIT_REPOSITORY "https://github.com/gflags/gflags.git" PREFIX ${GFLAGS_SOURCES_DIR} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR} + CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON CMAKE_ARGS -DBUILD_TESTING=OFF LOG_DOWNLOAD =ON UPDATE_COMMAND "" diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake index 8a4b9d5996f..473071a72a6 100644 --- a/cmake/external/glog.cmake +++ b/cmake/external/glog.cmake @@ -22,6 +22,7 @@ ExternalProject_Add( GIT_REPOSITORY "https://github.com/google/glog.git" PREFIX ${GLOG_SOURCES_DIR} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GLOG_INSTALL_DIR} + CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON CMAKE_ARGS -DWITH_GFLAGS=OFF CMAKE_ARGS -DBUILD_TESTING=OFF LOG_DOWNLOAD =ON diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake index 533104422ac..a6ed9e9b9f8 100644 --- a/cmake/external/gtest.cmake +++ b/cmake/external/gtest.cmake @@ -23,6 +23,7 @@ ExternalProject_Add( GIT_TAG "release-1.8.0" PREFIX ${GTEST_SOURCES_DIR} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR} + CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON CMAKE_ARGS -DBUILD_GMOCK=ON CMAKE_ARGS -Dgtest_disable_pthreads=ON CMAKE_ARGS -Dgtest_force_shared_crt=ON diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index efd07eb807a..f42e42ef681 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -36,22 +36,25 @@ SET(PROTOBUF_INCLUDE_DIR "${PROTOBUF_INSTALL_DIR}/include" CACHE PATH "protobuf INCLUDE_DIRECTORIES(${PROTOBUF_INCLUDE_DIR}) IF(WIN32) - SET(PROTOBUF_LIBRARIES - "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf-lite.lib" - "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf.lib" - "${PROTOBUF_INSTALL_DIR}/lib/libprotoc.lib" CACHE FILEPATH "protobuf libraries." FORCE) + SET(PROTOBUF_LITE_LIBRARY + "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf-lite.lib" CACHE FILEPATH "protobuf lite library." FORCE) + SET(PROTOBUF_LIBRARY + "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf.lib" CACHE FILEPATH "protobuf library." FORCE) + SET(PROTOBUF_PROTOC_LIBRARY + "${PROTOBUF_INSTALL_DIR}/lib/libprotoc.lib" CACHE FILEPATH "protoc library." FORCE) SET(PROTOBUF_PROTOC_EXECUTABLE "${PROTOBUF_INSTALL_DIR}/bin/protoc.exe" CACHE FILEPATH "protobuf executable." FORCE) ELSE(WIN32) FIND_PATH(PROTOBUF_LIBS_DIR libprotoc.a - PATHS - ${PROTOBUF_INSTALL_DIR}/lib64 ${PROTOBUF_INSTALL_DIR}/lib + ${PROTOBUF_INSTALL_DIR}/lib64 NO_DEFAULT_PATH ) - SET(PROTOBUF_LIBRARIES - "${PROTOBUF_LIBS_DIR}/libprotobuf-lite.a" - "${PROTOBUF_LIBS_DIR}/libprotobuf.a" - "${PROTOBUF_LIBS_DIR}/libprotoc.a" CACHE FILEPATH "protobuf libraries." FORCE) + SET(PROTOBUF_LITE_LIBRARY + "${PROTOBUF_LIBS_DIR}/libprotobuf-lite.a" CACHE FILEPATH "protobuf lite library." FORCE) + SET(PROTOBUF_LIBRARY + "${PROTOBUF_LIBS_DIR}/libprotobuf.a" CACHE FILEPATH "protobuf library." FORCE) + SET(PROTOBUF_PROTOC_LIBRARY + "${PROTOBUF_LIBS_DIR}/libprotoc.a" CACHE FILEPATH "protoc library." FORCE) SET(PROTOBUF_PROTOC_EXECUTABLE "${PROTOBUF_INSTALL_DIR}/bin/protoc" CACHE FILEPATH "protobuf executable." FORCE) ENDIF(WIN32) diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index b3dea19cebd..6a88c87df64 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -36,6 +36,7 @@ ExternalProject_Add( SET(WARPCTC_INCLUDE_DIR "${WARPCTC_INSTALL_DIR}/include" CACHE PATH "Warp-ctc Directory" FORCE) INCLUDE_DIRECTORIES(${WARPCTC_INCLUDE_DIR}) +SET(WARPCTC_LIB_DIR "${WARPCTC_INSTALL_DIR}/lib" CACHE PATH "Warp-ctc Library Directory" FORCE) IF(WIN32) SET(WARPCTC_LIBRARIES "${WARPCTC_INSTALL_DIR}/lib/warpctc.dll" CACHE FILEPATH "Warp-ctc Library" FORCE) diff --git a/paddle/api/paddle_api_config.py.in b/paddle/api/paddle_api_config.py.in index 23542b952b7..e11ee920362 100644 --- a/paddle/api/paddle_api_config.py.in +++ b/paddle/api/paddle_api_config.py.in @@ -1,17 +1,17 @@ PADDLE_BUILD_DIR="@CMAKE_CURRENT_BINARY_DIR@/../" WITH_GPU="@WITH_GPU@" -PROTOBUF_LIB="@PROTOBUF_LIBRARY@" -ZLIB_LIB="@ZLIB_LIBRARIES@" +PROTOBUF_LIBRARY="@PROTOBUF_LIBRARY@" +ZLIB_LIBRARIES="@ZLIB_LIBRARIES@" CMAKE_THREAD_LIB="@CMAKE_THREAD_LIBS_INIT@" CMAKE_DL_LIBS="@CMAKE_DL_LIBS@" WITH_PYTHON="@WITH_PYTHON@" PYTHON_LIBRARIES="@PYTHON_LIBRARIES@" -LIBGLOG_LIBRARY="@LIBGLOG_LIBRARY@" +GLOG_LIBRARIES="@GLOG_LIBRARIES@" GFLAGS_LIBRARIES="@GFLAGS_LIBRARIES@" GFLAGS_LOCATION="@GFLAGS_LOCATION@" -CBLAS_LIBRARIES="@CBLAS_LIBS@" +CBLAS_LIBRARIES="@CBLAS_LIBRARIES@" -CUDA_LIBRARIES="@CUDA_LIBRARIES@" +CUDA_LIBRARIES="@CUDA_cudart_shared_LIBRARY@" WITH_COVERALLS="@ON_COVERALLS@" diff --git a/paddle/api/paddle_ld_flags.py b/paddle/api/paddle_ld_flags.py index b4d27b1cc72..ad5dce209bf 100644 --- a/paddle/api/paddle_ld_flags.py +++ b/paddle/api/paddle_ld_flags.py @@ -40,14 +40,14 @@ try: self.paddle_build_dir = PADDLE_BUILD_DIR self.paddle_build_dir = os.path.abspath(self.paddle_build_dir) self.with_gpu = PaddleLDFlag.cmake_bool(WITH_GPU) - self.protolib = PROTOBUF_LIB - self.zlib = ZLIB_LIB + self.protolib = PROTOBUF_LIBRARY + self.zlib = ZLIB_LIBRARIES self.thread = CMAKE_THREAD_LIB self.dl_libs = CMAKE_DL_LIBS self.with_python = PaddleLDFlag.cmake_bool(WITH_PYTHON) self.python_libs = PYTHON_LIBRARIES - self.glog_libs = LIBGLOG_LIBRARY + self.glog_libs = GLOG_LIBRARIES self.with_coverage = PaddleLDFlag.cmake_bool(WITH_COVERALLS) self.gflags_libs = GFLAGS_LIBRARIES diff --git a/paddle/cuda/CMakeLists.txt b/paddle/cuda/CMakeLists.txt index 57fb89608f4..0a058978540 100755 --- a/paddle/cuda/CMakeLists.txt +++ b/paddle/cuda/CMakeLists.txt @@ -88,7 +88,7 @@ else() ${CUDA_CXX_SOURCES}) endif() -add_dependencies(paddle_cuda ${external_project_dependencies}) +add_dependencies(paddle_cuda warpctc) add_style_check_target(paddle_cuda ${CUDA_SOURCES} diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt index c26a2a7f06b..4190892db1f 100644 --- a/paddle/gserver/tests/CMakeLists.txt +++ b/paddle/gserver/tests/CMakeLists.txt @@ -92,7 +92,7 @@ if(NOT WITH_DOUBLE) TestUtil.cpp) add_test(NAME test_WarpCTCLayer - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_WarpCTCLayer --warpctc_dir=${PROJ_ROOT}/warp-ctc/build + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_WarpCTCLayer --warpctc_dir=${WARPCTC_LIB_DIR} WORKING_DIRECTORY ${PROJ_ROOT}/paddle) endif() diff --git a/paddle/scripts/travis/before_install.linux.sh b/paddle/scripts/travis/before_install.linux.sh deleted file mode 100755 index 9620bff6bcf..00000000000 --- a/paddle/scripts/travis/before_install.linux.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -e -pip install protobuf -cd /tmp -wget https://github.com/google/protobuf/archive/v3.0.2.tar.gz -O protobuf.tar.gz -tar xf protobuf.tar.gz -cd protobuf* -./autogen.sh -./configure --prefix=/usr/ -make -j 2 install -cd .. -rm -rf protobuf* - -pushd /usr/src/gtest -cmake . -make -sudo cp *.a /usr/lib -popd diff --git a/paddle/scripts/travis/before_install.osx.sh b/paddle/scripts/travis/before_install.osx.sh index bd88ed39132..89742d67f54 100755 --- a/paddle/scripts/travis/before_install.osx.sh +++ b/paddle/scripts/travis/before_install.osx.sh @@ -3,10 +3,4 @@ brew update brew tap homebrew/science brew install python sudo pip install --upgrade protobuf -brew install cmake python glog gflags openblas wget md5sha1sum protobuf - -wget https://github.com/google/googletest/archive/release-1.8.0.tar.gz -O gtest.tar.gz -tar xf gtest.tar.gz -cd googletest-release-1.8.0/ -cmake . -make install +brew install cmake python wget md5sha1sum diff --git a/paddle/scripts/travis/build_submodules.sh b/paddle/scripts/travis/build_submodules.sh deleted file mode 100755 index d458bf92bf4..00000000000 --- a/paddle/scripts/travis/build_submodules.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -e -WORK_DIR=$PWD -PROJ_ROOT=$(git rev-parse --show-cdup) -SUBMODULES=$(grep path ${PROJ_ROOT}.gitmodules | sed 's/^.*path = //') - -for module in $SUBMODULES -do - case $module in - "warp-ctc") - if [ -d ${PROJ_ROOT}warp-ctc/build ]; then - rm -rf ${PROJ_ROOT}warp-ctc/build - fi - mkdir ${PROJ_ROOT}warp-ctc/build - cd ${PROJ_ROOT}warp-ctc/build - cmake ..; make - ;; - esac -done -cd $WORK_DIR -- GitLab