From ff9fe6fa4a611875858ba23b64013cdf344f3d30 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 24 Aug 2020 15:30:52 +0800 Subject: [PATCH] Milvus build stage parallel processing (#3423) * Milvus build stage parallel processing Signed-off-by: quicksilver * Milvus build stage parallel processing Signed-off-by: quicksilver * Milvus build stage parallel processing Signed-off-by: quicksilver * Milvus build stage parallel processing Signed-off-by: quicksilver * Milvus build stage parallel processing Signed-off-by: quicksilver * Milvus build stage parallel processing Signed-off-by: quicksilver --- .github/workflows/cpplint.yml | 64 +++++++++++++++++++++++ .jenkins/modules/Build/MilvusBuild.groovy | 4 +- ci/scripts/build.sh | 23 ++++---- docker-compose.yml | 57 ++++++++++++++++++-- 4 files changed, 132 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/cpplint.yml diff --git a/.github/workflows/cpplint.yml b/.github/workflows/cpplint.yml new file mode 100644 index 00000000..79281be4 --- /dev/null +++ b/.github/workflows/cpplint.yml @@ -0,0 +1,64 @@ +name: Cpplint + +# This workflow is triggered on pushes or pull request to the repository. +on: + push: + # file paths to consider in the event. Optional; defaults to all. + paths: + - 'ci/**' + - 'core/**' + - 'docker/build_env/cpu/**' + - '.github/workflows/cpplint.yml' + - docker-compose.yml + - '!**.md' + - '!ci/jenkins/**' + pull_request: + # file paths to consider in the event. Optional; defaults to all. + paths: + - 'ci/**' + - 'core/**' + - '.github/workflows/cpplint.yml' + - docker-compose.yml + - '!**.md' + - '!ci/jenkins/**' + +jobs: + centos: + name: ${{ matrix.title }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + name: + - cpplint + - cpplint-gpu + include: + - name: cpplint + service: cpplint + title: Milvus CppLint + - name: cpplint-gpu + service: cpplint-gpu + title: Milvus GPU Version CppLint + steps: + # This step checks out a copy of your repository. + - name: Checkout Milvus + uses: actions/checkout@v2 + - name: Docker Pull + shell: bash + run: | + docker-compose pull --ignore-pull-failures ${{ matrix.service }} + - name: Docker Build + shell: bash + run: | + docker-compose build ${{ matrix.service }} + - name: Docker Run + run: | + docker-compose run ${{ matrix.service }} + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' + continue-on-error: true + shell: bash + run: | + docker login -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push ${{ matrix.service }} diff --git a/.jenkins/modules/Build/MilvusBuild.groovy b/.jenkins/modules/Build/MilvusBuild.groovy index 9e687b1f..62e10b67 100644 --- a/.jenkins/modules/Build/MilvusBuild.groovy +++ b/.jenkins/modules/Build/MilvusBuild.groovy @@ -3,11 +3,11 @@ timeout(time: 60, unit: 'MINUTES') { def isTimeTriggeredBuild = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0 if (!isTimeTriggeredBuild) { sh ". ./before-install.sh && ./check_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache --cache_dir=\${CCACHE_DIR} -f \${CCACHE_COMPRESS_PACKAGE_FILE} || echo \"ccache files not found!\"" - sh ". ./before-install.sh && ./check_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/thirdparty --cache_dir=\${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} -f \${THIRDPARTY_COMPRESS_PACKAGE_FILE} || echo \"thirdparty files not found!\"" + sh ". ./before-install.sh && ./check_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/thirdparty --cache_dir=\${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} -f \${THIRDPARTY_COMPRESS_PACKAGE_FILE} || echo \"thirdparty files not found!\"" } if ("${BINARY_VERSION}" == "gpu") { - sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --coverage -l -g -u\"" + sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --coverage -g -l -u\"" } else { sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --coverage -l -u\"" } diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index c212a417..0b08c2b4 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -31,6 +31,7 @@ Usage: -g Building for the architecture of the GPU in the system --with_mkl Build with MKL (default: OFF) --with_fiu Build with FIU (default: OFF) + --tidy Run clang-tidy -c or --coverage Build Code Coverage -u or --tests Build unittest case -p or --privileges Install command with elevated privileges @@ -41,7 +42,7 @@ Usage: Use \"$0 --help\" for more information about a given command. " -ARGS=`getopt -o "i:t:s:j::lngcupvh" -l "install_prefix::,build_type::,custom_thirdparty::,jobs::,with_mkl,with_fiu,coverage,tests,privileges,help" -n "$0" -- "$@"` +ARGS=`getopt -o "i:t:s:j::lngcupvh" -l "install_prefix::,build_type::,custom_thirdparty::,jobs::,tidy,with_mkl,with_fiu,coverage,tests,privileges,help" -n "$0" -- "$@"` eval set -- "${ARGS}" @@ -73,6 +74,7 @@ while true ; do -g) echo "Building for the architecture of the GPU in the system..." ; GPU_VERSION="ON" ; shift ;; --with_mkl) echo "Build with MKL" ; WITH_MKL="ON" ; shift ;; --with_fiu) echo "Build with FIU" ; FIU_ENABLE="ON" ; shift ;; + --tidy) echo "Run clang-tidy" ; RUN_CLANG_TIDY="ON" ; shift ;; --coverage) echo "Build code coverage" ; BUILD_COVERAGE="ON" ; shift ;; -u|--tests) echo "Build unittest cases" ; BUILD_UNITTEST="ON" ; shift ;; -n) echo "No build and install step" ; COMPILE_BUILD="OFF" ; shift ;; @@ -101,6 +103,7 @@ COMPILE_BUILD=${COMPILE_BUILD:="ON"} GPU_VERSION=${GPU_VERSION:="OFF"} CUDA_ARCH=${CUDA_ARCH:="DEFAULT"} RUN_CPPLINT=${RUN_CPPLINT:="OFF"} +RUN_CLANG_TIDY=${RUN_CLANG_TIDY:="OFF"} WITH_MKL=${WITH_MKL:="OFF"} FIU_ENABLE=${FIU_ENABLE:="OFF"} PRIVILEGES=${PRIVILEGES:="OFF"} @@ -159,19 +162,17 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then exit 1 fi echo "clang-format check passed!" +fi +if [[ ${RUN_CLANG_TIDY} == "ON" ]]; then # clang-tidy check - if [[ ${GPU_VERSION} == "ON" ]]; then - make check-clang-tidy - if [ $? -ne 0 ]; then - echo "ERROR! clang-tidy check failed" - rm -f CMakeCache.txt - exit 1 - fi - echo "clang-tidy check passed!" - else - echo "CPU version skip clang-tidy check!" + make check-clang-tidy + if [ $? -ne 0 ]; then + echo "ERROR! clang-tidy check failed" + rm -f CMakeCache.txt + exit 1 fi + echo "clang-tidy check passed!" fi if [[ ${COMPILE_BUILD} == "ON" ]];then diff --git a/docker-compose.yml b/docker-compose.yml index 78c2116c..a27c8e55 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ version: '3.5' volumes: amd64-ubuntu-18.04-cache: amd64-centos-7-cache: + amd64-ubuntu-18.04-cuda-cache: amd64-centos-7-cuda-cache: x-ccache: &ccache @@ -40,7 +41,7 @@ services: networks: - milvus command: &ubuntu-command - ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u && + ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -u && /milvus/ci/scripts/run_unittest.sh -i ${MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=mysql && /milvus/ci/scripts/coverage.sh"] @@ -60,10 +61,28 @@ services: networks: - milvus command: ¢os-command - ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u && + ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -u && /milvus/ci/scripts/run_unittest.sh -i ${MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=mysql && /milvus/ci/scripts/coverage.sh"] + ubuntu-cuda-core: + image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-core + build: + context: . + dockerfile: docker/build_env/gpu/ubuntu${UBUNTU}/Dockerfile + cache_from: + - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-core + shm_size: 2G + environment: + <<: *ccache + volumes: &ubuntu-cuda-volumes + - .:/milvus:delegated + - ${ARCH}-ubuntu-${UBUNTU}-cuda-cache:/build:delegated + networks: + - milvus + command: &ubuntu-cuda-command + ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} -g"] + centos-cuda-core: image: ${REPO}:${ARCH}-centos-${CENTOS}-cuda-core build: @@ -80,7 +99,39 @@ services: networks: - milvus command: ¢os-cuda-command - ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} -l -g"] + ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} -g"] + + cpplint: + image: ${REPO}:${ARCH}-centos-${CENTOS}-core + build: + context: . + dockerfile: docker/build_env/cpu/centos${CENTOS}/Dockerfile + cache_from: + - ${REPO}:${ARCH}-centos-${CENTOS}-core + shm_size: 2G + volumes: + - .:/milvus:delegated + - ${ARCH}-centos-${CENTOS}-cache:/build:delegated + networks: + - milvus + command: + ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} -l --tidy -n"] + + cpplint-gpu: + image: ${REPO}:${ARCH}-centos-${CENTOS}-cuda-core + build: + context: . + dockerfile: docker/build_env/gpu/centos${CENTOS}/Dockerfile + cache_from: + - ${REPO}:${ARCH}-centos-${CENTOS}-cuda-core + shm_size: 2G + volumes: + - .:/milvus:delegated + - ${ARCH}-centos-${CENTOS}-cuda-cache:/build:delegated + networks: + - milvus + command: + ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} -l -g --tidy -n"] networks: milvus: -- GitLab