From d849d0d0f74883e4ce1a55bca62b7741f82102c4 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 21 Aug 2020 21:53:18 +0800 Subject: [PATCH] Optimize thirdparty download workflow (#3394) * Optimize thirdparty download workflow Signed-off-by: quicksilver * Optimize thirdparty download workflow Signed-off-by: quicksilver * Optimize thirdparty download workflow Signed-off-by: quicksilver * Optimize thirdparty download workflow Signed-off-by: quicksilver --- .jenkins/modules/Build/MilvusBuild.groovy | 8 +++- ci/scripts/before-install.sh | 5 ++- ci/scripts/build.sh | 21 ++++++++-- .../{check_ccache.sh => check_cache.sh} | 30 +++++++++------ .../{update_ccache.sh => update_cache.sh} | 38 +++++++++---------- 5 files changed, 65 insertions(+), 37 deletions(-) rename ci/scripts/{check_ccache.sh => check_cache.sh} (82%) rename ci/scripts/{update_ccache.sh => update_cache.sh} (75%) diff --git a/.jenkins/modules/Build/MilvusBuild.groovy b/.jenkins/modules/Build/MilvusBuild.groovy index ef524adbe..9e687b1f8 100644 --- a/.jenkins/modules/Build/MilvusBuild.groovy +++ b/.jenkins/modules/Build/MilvusBuild.groovy @@ -2,7 +2,8 @@ timeout(time: 60, unit: 'MINUTES') { dir ("ci/scripts") { def isTimeTriggeredBuild = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0 if (!isTimeTriggeredBuild) { - sh "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache || echo \"ccache files not found!\"" + sh ". ./before-install.sh && ./check_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache --cache_dir=\${CCACHE_DIR} -f \${CCACHE_COMPRESS_PACKAGE_FILE} || echo \"ccache files not found!\"" + sh ". ./before-install.sh && ./check_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/thirdparty --cache_dir=\${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} -f \${THIRDPARTY_COMPRESS_PACKAGE_FILE} || echo \"thirdparty files not found!\"" } if ("${BINARY_VERSION}" == "gpu") { @@ -12,7 +13,10 @@ timeout(time: 60, unit: 'MINUTES') { } withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}" + sh ". ./before-install.sh && ./update_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache --cache_dir=\${CCACHE_DIR} -f \${CCACHE_COMPRESS_PACKAGE_FILE} -u ${USERNAME} -p ${PASSWORD}" + if (isTimeTriggeredBuild) { + sh ". ./before-install.sh && ./update_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/thirdparty --cache_dir=\${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} -f \${THIRDPARTY_COMPRESS_PACKAGE_FILE} -u ${USERNAME} -p ${PASSWORD}" + } } } } diff --git a/ci/scripts/before-install.sh b/ci/scripts/before-install.sh index 5d045c486..e767de26c 100755 --- a/ci/scripts/before-install.sh +++ b/ci/scripts/before-install.sh @@ -7,6 +7,9 @@ export CCACHE_COMPRESSLEVEL=5 export CCACHE_COMPILERCHECK=content export PATH=/usr/lib/ccache/:$PATH export CCACHE_BASEDIR=${WORKSPACE:=""} +export CCACHE_DIR=${CCACHE_DIR:="${HOME}/.ccache"} +export CCACHE_COMPRESS_PACKAGE_FILE=${CCACHE_COMPRESS_PACKAGE_FILE:="ccache-${OS_NAME}-${BUILD_ENV_IMAGE_ID}.tar.gz"} +export CUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_DOWNLOAD_PATH:="${HOME}/3rdparty_download"} +export THIRDPARTY_COMPRESS_PACKAGE_FILE=${THIRDPARTY_COMPRESS_PACKAGE_FILE:="thirdparty-download.tar.gz"} set +ex - diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 50abb507b..fc33828ab 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -24,6 +24,8 @@ Usage: Build type (default: Release) -s [CUDA_ARCH] Building for the cuda architecture -j[N] or --jobs=[N] Allow N jobs at once; infinite jobs with no arg. + --custom_thirdparty=[CUSTOM_THIRDPARTY_PATH] + Custom paths of thirdparty downloaded files(default: NULL) -l Run cpplint & check clang-format -n No make and make install step -g Building for the architecture of the GPU in the system @@ -39,7 +41,7 @@ Usage: Use \"$0 --help\" for more information about a given command. " -ARGS=`getopt -o "i:t:s:j::lngcupvh" -l "install_prefix::,build_type::,jobs::,with_mkl,with_fiu,coverage,tests,privileges,help" -n "$0" -- "$@"` +ARGS=`getopt -o "i:t:s:j::lngcupvh" -l "install_prefix::,build_type::,custom_thirdparty::,jobs::,with_mkl,with_fiu,coverage,tests,privileges,help" -n "$0" -- "$@"` eval set -- "${ARGS}" @@ -63,6 +65,11 @@ while true ; do "") PARALLEL_LEVEL=""; shift 2 ;; *) PARALLEL_LEVEL=$2 ; shift 2 ;; esac ;; + --custom_thirdparty) + case "$2" in + "") CUSTOM_THIRDPARTY_DOWNLOAD_PATH=""; shift 2 ;; + *) CUSTOM_THIRDPARTY_DOWNLOAD_PATH=$2 ; shift 2 ;; + esac ;; -g) echo "Building for the architecture of the GPU in the system..." ; GPU_VERSION="ON" ; shift ;; --with_mkl) echo "Build with MKL" ; WITH_MKL="ON" ; shift ;; --with_fiu) echo "Build with FIU" ; FIU_ENABLE="ON" ; shift ;; @@ -99,6 +106,7 @@ FIU_ENABLE=${FIU_ENABLE:="OFF"} PRIVILEGES=${PRIVILEGES:="OFF"} CLEANUP=${CLEANUP:="OFF"} PARALLEL_LEVEL=${PARALLEL_LEVEL:="8"} +CUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_DOWNLOAD_PATH:=""} for arg do if [[ $arg == "clean" ]];then @@ -115,9 +123,6 @@ if [[ ! -d ${CORE_BUILD_DIR} ]]; then mkdir ${CORE_BUILD_DIR} fi -# Zero the cache statistics (but not the configuration options) -ccache -z - pushd ${CORE_BUILD_DIR} CMAKE_CMD="cmake \ @@ -133,6 +138,7 @@ CMAKE_CMD="cmake \ -DOpenBLAS_SOURCE=AUTO \ -DMILVUS_WITH_FIU=${FIU_ENABLE} \ -DMILVUS_CUDA_ARCH=${CUDA_ARCH} \ +-DCUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} \ ${MILVUS_CORE_DIR}" echo ${CMAKE_CMD} ${CMAKE_CMD} @@ -165,6 +171,10 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then fi if [[ ${COMPILE_BUILD} == "ON" ]];then + + # Zero the cache statistics (but not the configuration options) + ccache -z + # compile and build make -j${PARALLEL_LEVEL} VERBOSE=${VERBOSE} || exit 1 @@ -173,6 +183,9 @@ if [[ ${COMPILE_BUILD} == "ON" ]];then else make install || exit 1 fi + + echo -e "===\n=== ccache statistics after build\n===" + ccache --show-stats fi popd diff --git a/ci/scripts/check_ccache.sh b/ci/scripts/check_cache.sh similarity index 82% rename from ci/scripts/check_ccache.sh rename to ci/scripts/check_cache.sh index 61055b377..d67518174 100755 --- a/ci/scripts/check_ccache.sh +++ b/ci/scripts/check_cache.sh @@ -5,8 +5,8 @@ Usage: $0 [flags] [Arguments] -l [ARTIFACTORY_URL] Artifactory URL - --cache_dir=[CCACHE_DIR] Ccache directory - -f [FILE] or --file=[FILE] Ccache compress package file + --cache_dir=[CACHE_DIR] Cache directory + -f [FILE] or --file=[FILE] Cache compress package file -h or --help Print help information @@ -30,7 +30,7 @@ while true ; do --cache_dir) case "$2" in "") echo "Option cache_dir, no argument"; exit 1 ;; - *) CCACHE_DIR=$2 ; shift 2 ;; + *) CACHE_DIR=$2 ; shift 2 ;; esac ;; -f|--file) case "$2" in @@ -44,8 +44,6 @@ while true ; do done # Set defaults for vars modified by flags to this script -CCACHE_DIR=${CCACHE_DIR:="${HOME}/.ccache"} -PACKAGE_FILE=${PACKAGE_FILE:="ccache-${OS_NAME}-${BUILD_ENV_IMAGE_ID}.tar.gz"} BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then @@ -53,7 +51,17 @@ if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then exit 1 fi -function check_ccache() { +if [[ -z "${CACHE_DIR}" ]]; then + echo "You have not input CACHE_DIR !" + exit 1 +fi + +if [[ -z "${PACKAGE_FILE}" ]]; then + echo "You have not input PACKAGE_FILE !" + exit 1 +fi + +function check_cache() { BRANCH=$1 echo "fetching ${BRANCH}/${PACKAGE_FILE}" wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" @@ -63,14 +71,14 @@ function check_ccache() { function download_file() { BRANCH=$1 wget -q "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" && \ - mkdir -p "${CCACHE_DIR}" && \ - tar zxf "${PACKAGE_FILE}" -C "${CCACHE_DIR}" && \ + mkdir -p "${CACHE_DIR}" && \ + tar zxf "${PACKAGE_FILE}" -C "${CACHE_DIR}" && \ rm ${PACKAGE_FILE} return $? } if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]];then - check_ccache ${CHANGE_TARGET} + check_cache ${CHANGE_TARGET} if [[ $? == 0 ]];then download_file ${CHANGE_TARGET} if [[ $? == 0 ]];then @@ -79,7 +87,7 @@ if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]];then fi fi - check_ccache ${BRANCH_NAME} + check_cache ${BRANCH_NAME} if [[ $? == 0 ]];then download_file ${BRANCH_NAME} if [[ $? == 0 ]];then @@ -92,7 +100,7 @@ fi for CURRENT_BRANCH in ${BRANCH_NAMES} do if [[ "${CURRENT_BRANCH}" != "HEAD" ]];then - check_ccache ${CURRENT_BRANCH} + check_cache ${CURRENT_BRANCH} if [[ $? == 0 ]];then download_file ${CURRENT_BRANCH} if [[ $? == 0 ]];then diff --git a/ci/scripts/update_ccache.sh b/ci/scripts/update_cache.sh similarity index 75% rename from ci/scripts/update_ccache.sh rename to ci/scripts/update_cache.sh index bdc51f0bd..85985b741 100755 --- a/ci/scripts/update_ccache.sh +++ b/ci/scripts/update_cache.sh @@ -5,8 +5,8 @@ Usage: $0 [flags] [Arguments] -l [ARTIFACTORY_URL] Artifactory URL - --cache_dir=[CCACHE_DIR] Ccache directory - -f [FILE] or --file=[FILE] Ccache compress package file + --cache_dir=[CACHE_DIR] Cache directory + -f [FILE] or --file=[FILE] Cache compress package file -u [USERNAME] Artifactory Username -p [PASSWORD] Artifactory Password -h or --help Print help information @@ -32,7 +32,7 @@ while true ; do --cache_dir) case "$2" in "") echo "Option cache_dir, no argument"; exit 1 ;; - *) CCACHE_DIR=$2 ; shift 2 ;; + *) CACHE_DIR=$2 ; shift 2 ;; esac ;; -u) case "$2" in @@ -56,8 +56,6 @@ while true ; do done # Set defaults for vars modified by flags to this script -CCACHE_DIR=${CCACHE_DIR:="${HOME}/.ccache"} -PACKAGE_FILE=${PACKAGE_FILE:="ccache-${OS_NAME}-${BUILD_ENV_IMAGE_ID}.tar.gz"} BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g') if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then @@ -65,40 +63,42 @@ if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then exit 1 fi -if [[ ! -d "${CCACHE_DIR}" ]]; then - echo "\"${CCACHE_DIR}\" directory does not exist !" +if [[ ! -d "${CACHE_DIR}" ]]; then + echo "\"${CACHE_DIR}\" directory does not exist !" exit 1 fi -function check_ccache() { +if [[ -z "${PACKAGE_FILE}" ]]; then + echo "You have not input PACKAGE_FILE !" + exit 1 +fi + +function check_cache() { BRANCH=$1 wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" return $? } -echo -e "===\n=== ccache statistics after build\n===" -ccache --show-stats - if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]]; then - check_ccache ${CHANGE_TARGET} + check_cache ${CHANGE_TARGET} if [[ $? == 0 ]];then - echo "Skip Update ccache package ..." && exit 0 + echo "Skip Update cache package ..." && exit 0 fi fi if [[ "${BRANCH_NAME}" != "HEAD" ]];then REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}" - echo "Updating ccache package file: ${PACKAGE_FILE}" - tar zcf ./"${PACKAGE_FILE}" -C "${CCACHE_DIR}" . - echo "Uploading ccache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}" + echo "Updating cache package file: ${PACKAGE_FILE}" + tar zcf ./"${PACKAGE_FILE}" -C "${CACHE_DIR}" . + echo "Uploading cache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}" curl -u"${USERNAME}":"${PASSWORD}" -T "${PACKAGE_FILE}" "${REMOTE_PACKAGE_PATH}"/"${PACKAGE_FILE}" if [[ $? == 0 ]];then - echo "Uploading ccache package file success !" + echo "Uploading cache package file success !" exit 0 else - echo "Uploading ccache package file fault !" + echo "Uploading cache package file fault !" exit 1 fi fi -echo "Skip Update ccache package ..." +echo "Skip Update cache package ..." -- GitLab