Optimize thirdparty download workflow (#3394)

* Optimize thirdparty download workflow
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Optimize thirdparty download workflow
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Optimize thirdparty download workflow
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Optimize thirdparty download workflow
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

.jenkins/modules/Build/MilvusBuild.groovy

@@ -2,7 +2,8 @@ timeout(time: 60, unit: 'MINUTES') {
 	dir ("ci/scripts") {
 		def isTimeTriggeredBuild = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0
 		if (!isTimeTriggeredBuild) {
-			sh "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache || echo \"ccache files not found!\""
+			sh ". ./before-install.sh && ./check_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache --cache_dir=\${CCACHE_DIR} -f \${CCACHE_COMPRESS_PACKAGE_FILE} || echo \"ccache files not found!\""
+		    sh ". ./before-install.sh && ./check_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/thirdparty --cache_dir=\${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} -f \${THIRDPARTY_COMPRESS_PACKAGE_FILE} || echo \"thirdparty files not found!\""
 		}
 
 		if ("${BINARY_VERSION}" == "gpu") {
@@ -12,7 +13,10 @@ timeout(time: 60, unit: 'MINUTES') {
 		}
 
 		withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-			sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}"
+			sh ". ./before-install.sh && ./update_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache --cache_dir=\${CCACHE_DIR} -f \${CCACHE_COMPRESS_PACKAGE_FILE} -u ${USERNAME} -p ${PASSWORD}"
+			if (isTimeTriggeredBuild) {
+				sh ". ./before-install.sh && ./update_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/thirdparty --cache_dir=\${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} -f \${THIRDPARTY_COMPRESS_PACKAGE_FILE} -u ${USERNAME} -p ${PASSWORD}"
+			}
 		}
 	}
 }

ci/scripts/before-install.sh

@@ -7,6 +7,9 @@ export CCACHE_COMPRESSLEVEL=5
 export CCACHE_COMPILERCHECK=content
 export PATH=/usr/lib/ccache/:$PATH
 export CCACHE_BASEDIR=${WORKSPACE:=""}
+export CCACHE_DIR=${CCACHE_DIR:="${HOME}/.ccache"}
+export CCACHE_COMPRESS_PACKAGE_FILE=${CCACHE_COMPRESS_PACKAGE_FILE:="ccache-${OS_NAME}-${BUILD_ENV_IMAGE_ID}.tar.gz"}
+export CUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_DOWNLOAD_PATH:="${HOME}/3rdparty_download"}
+export THIRDPARTY_COMPRESS_PACKAGE_FILE=${THIRDPARTY_COMPRESS_PACKAGE_FILE:="thirdparty-download.tar.gz"}
 
 set +ex
-

ci/scripts/build.sh

@@ -24,6 +24,8 @@ Usage:
                               Build type (default: Release)
     -s [CUDA_ARCH]            Building for the cuda architecture
     -j[N] or --jobs=[N]       Allow N jobs at once; infinite jobs with no arg.
+    --custom_thirdparty=[CUSTOM_THIRDPARTY_PATH]
+                              Custom paths of thirdparty downloaded files(default: NULL)
     -l                        Run cpplint & check clang-format
     -n                        No make and make install step
     -g                        Building for the architecture of the GPU in the system
@@ -39,7 +41,7 @@ Usage:
 Use \"$0  --help\" for more information about a given command.
 "
 
-ARGS=`getopt -o "i:t:s:j::lngcupvh" -l "install_prefix::,build_type::,jobs::,with_mkl,with_fiu,coverage,tests,privileges,help" -n "$0" -- "$@"`
+ARGS=`getopt -o "i:t:s:j::lngcupvh" -l "install_prefix::,build_type::,custom_thirdparty::,jobs::,with_mkl,with_fiu,coverage,tests,privileges,help" -n "$0" -- "$@"`
 
 eval set -- "${ARGS}"
 
@@ -63,6 +65,11 @@ while true ; do
                                 "") PARALLEL_LEVEL=""; shift 2 ;;
                                 *)  PARALLEL_LEVEL=$2 ; shift 2 ;;
                         esac ;;
+                --custom_thirdparty)
+                        case "$2" in
+                                "") CUSTOM_THIRDPARTY_DOWNLOAD_PATH=""; shift 2 ;;
+                                *)  CUSTOM_THIRDPARTY_DOWNLOAD_PATH=$2 ; shift 2 ;;
+                        esac ;;
                 -g) echo "Building for the architecture of the GPU in the system..." ; GPU_VERSION="ON" ; shift ;;
                 --with_mkl) echo "Build with MKL" ; WITH_MKL="ON" ; shift ;;
                 --with_fiu) echo "Build with FIU" ; FIU_ENABLE="ON" ; shift ;;
@@ -99,6 +106,7 @@ FIU_ENABLE=${FIU_ENABLE:="OFF"}
 PRIVILEGES=${PRIVILEGES:="OFF"}
 CLEANUP=${CLEANUP:="OFF"}
 PARALLEL_LEVEL=${PARALLEL_LEVEL:="8"}
+CUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_DOWNLOAD_PATH:=""}
 
 for arg do
 if [[ $arg == "clean" ]];then
@@ -115,9 +123,6 @@ if [[ ! -d ${CORE_BUILD_DIR} ]]; then
     mkdir ${CORE_BUILD_DIR}
 fi
 
-# Zero the cache statistics (but not the configuration options)
-ccache -z
-
 pushd ${CORE_BUILD_DIR}
 
 CMAKE_CMD="cmake \
@@ -133,6 +138,7 @@ CMAKE_CMD="cmake \
 -DOpenBLAS_SOURCE=AUTO \
 -DMILVUS_WITH_FIU=${FIU_ENABLE} \
 -DMILVUS_CUDA_ARCH=${CUDA_ARCH} \
+-DCUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} \
 ${MILVUS_CORE_DIR}"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}
@@ -165,6 +171,10 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
 fi
 
 if [[ ${COMPILE_BUILD} == "ON" ]];then
+
+    # Zero the cache statistics (but not the configuration options)
+    ccache -z
+
     # compile and build
     make -j${PARALLEL_LEVEL} VERBOSE=${VERBOSE} || exit 1
 
@@ -173,6 +183,9 @@ if [[ ${COMPILE_BUILD} == "ON" ]];then
     else
         make install || exit 1
     fi
+
+    echo -e "===\n=== ccache statistics after build\n==="
+    ccache --show-stats
 fi
 
 popd

ci/scripts/check_ccache.sh → ci/scripts/check_cache.sh

@@ -5,8 +5,8 @@ Usage:
   $0 [flags] [Arguments]
 
     -l [ARTIFACTORY_URL]          Artifactory URL
-    --cache_dir=[CCACHE_DIR]      Ccache directory
-    -f [FILE] or --file=[FILE]    Ccache compress package file
+    --cache_dir=[CACHE_DIR]       Cache directory
+    -f [FILE] or --file=[FILE]    Cache compress package file
     -h or --help                  Print help information
 
 
@@ -30,7 +30,7 @@ while true ; do
                 --cache_dir)
                         case "$2" in
                                 "") echo "Option cache_dir, no argument"; exit 1 ;;
-                                *)  CCACHE_DIR=$2 ; shift 2 ;;
+                                *)  CACHE_DIR=$2 ; shift 2 ;;
                         esac ;;
                 -f|--file)
                         case "$2" in
@@ -44,8 +44,6 @@ while true ; do
 done
 
 # Set defaults for vars modified by flags to this script
-CCACHE_DIR=${CCACHE_DIR:="${HOME}/.ccache"}
-PACKAGE_FILE=${PACKAGE_FILE:="ccache-${OS_NAME}-${BUILD_ENV_IMAGE_ID}.tar.gz"}
 BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}')
 
 if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then
@@ -53,7 +51,17 @@ if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then
     exit 1
 fi
 
-function check_ccache() {
+if [[ -z "${CACHE_DIR}" ]]; then
+    echo "You have not input CACHE_DIR !"
+    exit 1
+fi
+
+if [[ -z "${PACKAGE_FILE}" ]]; then
+    echo "You have not input PACKAGE_FILE !"
+    exit 1
+fi
+
+function check_cache() {
     BRANCH=$1
     echo "fetching ${BRANCH}/${PACKAGE_FILE}"
     wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}"
@@ -63,14 +71,14 @@ function check_ccache() {
 function download_file() {
     BRANCH=$1
     wget -q "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" && \
-    mkdir -p "${CCACHE_DIR}" && \
-    tar zxf "${PACKAGE_FILE}" -C "${CCACHE_DIR}" && \
+    mkdir -p "${CACHE_DIR}" && \
+    tar zxf "${PACKAGE_FILE}" -C "${CACHE_DIR}" && \
     rm ${PACKAGE_FILE}
     return $?
 }
 
 if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]];then
-    check_ccache ${CHANGE_TARGET}
+    check_cache ${CHANGE_TARGET}
     if [[ $? == 0 ]];then
         download_file ${CHANGE_TARGET}
         if [[ $? == 0 ]];then
@@ -79,7 +87,7 @@ if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]];then
         fi
     fi
 
-    check_ccache ${BRANCH_NAME}
+    check_cache ${BRANCH_NAME}
     if [[ $? == 0 ]];then
         download_file ${BRANCH_NAME}
         if [[ $? == 0 ]];then
@@ -92,7 +100,7 @@ fi
 for CURRENT_BRANCH in ${BRANCH_NAMES}
 do
     if [[ "${CURRENT_BRANCH}" != "HEAD" ]];then
-        check_ccache ${CURRENT_BRANCH}
+        check_cache ${CURRENT_BRANCH}
         if [[ $? == 0 ]];then
             download_file ${CURRENT_BRANCH}
             if [[ $? == 0 ]];then

ci/scripts/update_ccache.sh → ci/scripts/update_cache.sh

@@ -5,8 +5,8 @@ Usage:
   $0 [flags] [Arguments]
 
     -l [ARTIFACTORY_URL]          Artifactory URL
-    --cache_dir=[CCACHE_DIR]      Ccache directory
-    -f [FILE] or --file=[FILE]    Ccache compress package file
+    --cache_dir=[CACHE_DIR]       Cache directory
+    -f [FILE] or --file=[FILE]    Cache compress package file
     -u [USERNAME]                 Artifactory Username
     -p [PASSWORD]                 Artifactory Password
     -h or --help                  Print help information
@@ -32,7 +32,7 @@ while true ; do
                 --cache_dir)
                         case "$2" in
                                 "") echo "Option cache_dir, no argument"; exit 1 ;;
-                                *)  CCACHE_DIR=$2 ; shift 2 ;;
+                                *)  CACHE_DIR=$2 ; shift 2 ;;
                         esac ;;
                 -u)
                         case "$2" in
@@ -56,8 +56,6 @@ while true ; do
 done
 
 # Set defaults for vars modified by flags to this script
-CCACHE_DIR=${CCACHE_DIR:="${HOME}/.ccache"}
-PACKAGE_FILE=${PACKAGE_FILE:="ccache-${OS_NAME}-${BUILD_ENV_IMAGE_ID}.tar.gz"}
 BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g')
 
 if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then
@@ -65,40 +63,42 @@ if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then
     exit 1
 fi
 
-if [[ ! -d "${CCACHE_DIR}" ]]; then
-    echo "\"${CCACHE_DIR}\" directory does not exist !"
+if [[ ! -d "${CACHE_DIR}" ]]; then
+    echo "\"${CACHE_DIR}\" directory does not exist !"
     exit 1
 fi
 
-function check_ccache() {
+if [[ -z "${PACKAGE_FILE}" ]]; then
+    echo "You have not input PACKAGE_FILE !"
+    exit 1
+fi
+
+function check_cache() {
     BRANCH=$1
     wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}"
     return $?
 }
 
-echo -e "===\n=== ccache statistics after build\n==="
-ccache --show-stats
-
 if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]]; then
-    check_ccache ${CHANGE_TARGET}
+    check_cache ${CHANGE_TARGET}
     if [[ $? == 0 ]];then
-        echo "Skip Update ccache package ..." && exit 0
+        echo "Skip Update cache package ..." && exit 0
     fi
 fi
 
 if [[ "${BRANCH_NAME}" != "HEAD" ]];then
     REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}"
-    echo "Updating ccache package file: ${PACKAGE_FILE}"
-    tar zcf ./"${PACKAGE_FILE}" -C "${CCACHE_DIR}" .
-    echo "Uploading ccache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}"
+    echo "Updating cache package file: ${PACKAGE_FILE}"
+    tar zcf ./"${PACKAGE_FILE}" -C "${CACHE_DIR}" .
+    echo "Uploading cache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}"
     curl -u"${USERNAME}":"${PASSWORD}" -T "${PACKAGE_FILE}" "${REMOTE_PACKAGE_PATH}"/"${PACKAGE_FILE}"
     if [[ $? == 0 ]];then
-        echo "Uploading ccache package file success !"
+        echo "Uploading cache package file success !"
         exit 0
     else
-        echo "Uploading ccache package file fault !"
+        echo "Uploading cache package file fault !"
         exit 1
     fi
 fi
 
-echo "Skip Update ccache package ..."
+echo "Skip Update cache package ..."