Remove Jfrog Cache on Jenkins CI (#827)

* specify multiple urls on sqlite_orm download stage

* fix bug

* fix bug

* specify multiple urls on opentracing download stage

* fix bug

* specify multiple urls on download stage

* delete jfrog cache

* print jenkins enviroment variables

* print jenkins enviroment variables

* fix check_ccache.sh bug

* debug

* Update Jenkinfile

* Add build enviroment resource limit on Jenkins CI

* remove Jfrog cache build option

ci/jenkins/Jenkinsfile

@@ -133,8 +133,8 @@ pipeline {
 
                             stage('Deploy to Development') {
                                 environment {
-                                    FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-")
-                                    FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-")
+                                    FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-").replaceAll("_", "-")
+                                    FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-")
                                     HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
                                 }
 

ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml

@@ -25,7 +25,7 @@ spec:
     resources:
       limits:
         memory: "12Gi"
-        cpu: "4.0"
+        cpu: "6.0"
       requests:
         memory: "8Gi"
         cpu: "4.0"

ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml

@@ -24,7 +24,7 @@ spec:
     resources:
       limits:
         memory: "12Gi"
-        cpu: "4.0"
+        cpu: "6.0"
       requests:
         memory: "8Gi"
         cpu: "4.0"

ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml

@@ -25,7 +25,7 @@ spec:
     resources:
       limits:
         memory: "12Gi"
-        cpu: "4.0"
+        cpu: "6.0"
         nvidia.com/gpu: 1
       requests:
         memory: "8Gi"

ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml

@@ -24,7 +24,7 @@ spec:
     resources:
       limits:
         memory: "12Gi"
-        cpu: "4.0"
+        cpu: "6.0"
         nvidia.com/gpu: 1
       requests:
         memory: "8Gi"

ci/scripts/build.sh

@@ -19,13 +19,12 @@ FAISS_ROOT=""
 PRIVILEGES="OFF"
 CUSTOMIZATION="OFF" # default use origin faiss
 BUILD_COVERAGE="OFF"
-USE_JFROG_CACHE="OFF"
 RUN_CPPLINT="OFF"
 GPU_VERSION="OFF"
 WITH_MKL="OFF"
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 
-while getopts "o:t:b:f:pgxulcjmh" arg
+while getopts "o:t:b:f:pgxulcmh" arg
 do
         case $arg in
              o)
@@ -59,9 +58,6 @@ do
              c)
                 BUILD_COVERAGE="ON"
                 ;;
-             j)
-                USE_JFROG_CACHE="ON"
-                ;;
              m)
                 WITH_MKL="ON"
                 ;;
@@ -79,12 +75,11 @@ parameter:
 -u: building unit test options(default: OFF)
 -l: run cpplint, clang-format and clang-tidy(default: OFF)
 -c: code coverage(default: OFF)
--j: use jfrog cache build directory(default: OFF)
 -m: build with MKL(default: OFF)
 -h: help
 
 usage:
-./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h]
+./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-x] [-u] [-l] [-c] [-m] [-h]
                 "
                 exit 0
                 ;;
@@ -112,7 +107,6 @@ CMAKE_CMD="cmake \
 -DCUSTOMIZATION=${CUSTOMIZATION} \
 -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
--DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
 -DFAISS_ROOT=${FAISS_ROOT} \
 -DFAISS_WITH_MKL=${WITH_MKL} \
 -DArrow_SOURCE=AUTO \

ci/scripts/check_ccache.sh

@@ -57,8 +57,8 @@ check_ccache() {
     fi
 }
 
-if [[ -n "${CHANGE_BRANCH}" && "${BRANCH_NAME}" =~ "PR-" ]];then
-    check_ccache ${CHANGE_BRANCH}
+if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]];then
+    check_ccache ${CHANGE_TARGET}
     check_ccache ${BRANCH_NAME}
 fi
 

core/build.sh

@@ -8,7 +8,6 @@ MAKE_CLEAN="OFF"
 BUILD_COVERAGE="OFF"
 DB_PATH="/tmp/milvus"
 PROFILING="OFF"
-USE_JFROG_CACHE="OFF"
 RUN_CPPLINT="OFF"
 CUSTOMIZATION="OFF" # default use ori faiss
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
@@ -19,7 +18,7 @@ FAISS_SOURCE="BUNDLED"
 WITH_PROMETHEUS="ON"
 FIU_ENABLE="OFF"
 
-while getopts "p:d:t:f:ulrcgjhxzmei" arg; do
+while getopts "p:d:t:f:ulrcghxzmei" arg; do
   case $arg in
   p)
     INSTALL_PREFIX=$OPTARG
@@ -53,9 +52,6 @@ while getopts "p:d:t:f:ulrcgjhxzmei" arg; do
   z)
     PROFILING="ON"
     ;;
-  j)
-    USE_JFROG_CACHE="ON"
-    ;;
   x)
     CUSTOMIZATION="ON"
     ;;
@@ -86,7 +82,6 @@ parameter:
 -r: remove previous build directory(default: OFF)
 -c: code coverage(default: OFF)
 -z: profiling(default: OFF)
--j: use jfrog cache build directory(default: OFF)
 -g: build GPU version(default: OFF)
 -m: build with MKL(default: OFF)
 -e: build without prometheus(default: OFF)
@@ -94,7 +89,7 @@ parameter:
 -h: help
 
 usage:
-./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-e] [-h]
+./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-g] [-m] [-e] [-h]
                 "
     exit 0
     ;;
@@ -125,7 +120,6 @@ CMAKE_CMD="cmake \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
 -DMILVUS_DB_PATH=${DB_PATH} \
 -DMILVUS_ENABLE_PROFILING=${PROFILING} \
--DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
 -DCUSTOMIZATION=${CUSTOMIZATION} \
 -DMILVUS_GPU_VERSION=${GPU_VERSION} \
 -DFAISS_WITH_MKL=${WITH_MKL} \

core/src/index/build.sh

@@ -6,9 +6,8 @@ INSTALL_PREFIX=$(pwd)/cmake_build
 MAKE_CLEAN="OFF"
 PROFILING="OFF"
 FAISS_WITH_MKL="OFF"
-USE_JFROG_CACHE="OFF"
 
-while getopts "p:d:t:uhrcgmj" arg
+while getopts "p:d:t:uhrcgm" arg
 do
         case $arg in
              t)
@@ -33,9 +32,6 @@ do
              m)
                 FAISS_WITH_MKL="ON"
                 ;;
-             j)
-                USE_JFROG_CACHE="ON"
-                ;;
              h) # help
                 echo "
 
@@ -75,7 +71,6 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
     -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
     -DMILVUS_ENABLE_PROFILING=${PROFILING} \
     -DFAISS_WITH_MKL=${FAISS_WITH_MKL} \
-    -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
     ../"
     echo ${CMAKE_CMD}
 

core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -115,20 +115,6 @@ endif (UNIX)
 # thirdparty directory
 set(THIRDPARTY_DIR "${INDEX_SOURCE_DIR}/thirdparty")
 
-# ----------------------------------------------------------------------
-# JFrog
-if (NOT DEFINED USE_JFROG_CACHE)
-    set(USE_JFROG_CACHE "OFF")
-endif ()
-if (USE_JFROG_CACHE STREQUAL "ON")
-    set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${KNOWHERE_BUILD_ARCH}/${BUILD_TYPE}")
-    set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache")
-    if (NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE})
-        message(STATUS "Will create cached directory: ${THIRDPARTY_PACKAGE_CACHE}")
-        file(MAKE_DIRECTORY ${THIRDPARTY_PACKAGE_CACHE})
-    endif ()
-endif ()
-
 # ----------------------------------------------------------------------
 # ExternalProject options
 
@@ -284,71 +270,25 @@ macro(build_arrow)
             -DBOOST_SOURCE=AUTO #try to find BOOST in the system default locations and build from source if not found
             )
 
-
-    if (USE_JFROG_CACHE STREQUAL "ON")
-        execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID)
-        if (${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
-            string(MD5 ARROW_COMBINE_MD5 "${ARROW_LAST_COMMIT_ID}")
-            set(ARROW_CACHE_PACKAGE_NAME "arrow_${ARROW_COMBINE_MD5}.tar.gz")
-            set(ARROW_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ARROW_CACHE_PACKAGE_NAME}")
-            set(ARROW_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ARROW_CACHE_PACKAGE_NAME}")
-
-            execute_process(COMMAND wget -q --method HEAD ${ARROW_CACHE_URL} RESULT_VARIABLE return_code)
-            message(STATUS "Check the remote file ${ARROW_CACHE_URL}. return code = ${return_code}")
-            if (NOT return_code EQUAL 0)
-                externalproject_add(arrow_ep
-                        GIT_REPOSITORY
-                        ${ARROW_SOURCE_URL}
-                        GIT_TAG
-                        ${ARROW_VERSION}
-                        GIT_SHALLOW
-                        TRUE
-                        SOURCE_SUBDIR
-                        cpp
-                        ${EP_LOG_OPTIONS}
-                        CMAKE_ARGS
-                        ${ARROW_CMAKE_ARGS}
-                        BUILD_COMMAND
-                        ""
-                        INSTALL_COMMAND
-                        ${MAKE} ${MAKE_BUILD_ARGS} install
-                        BUILD_BYPRODUCTS
-                        "${ARROW_STATIC_LIB}"
-                        )
-
-                ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL})
-            else ()
-                file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status)
-                list(GET status 0 status_code)
-                message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-                if (status_code EQUAL 0)
-                    ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR})
-                endif ()
-            endif ()
-        else ()
-            message(FATAL_ERROR "The last commit ID of \"${ARROW_SOURCE_URL}\" repository don't match!")
-        endif ()
-    else ()
-        externalproject_add(arrow_ep
-                GIT_REPOSITORY
-                ${ARROW_SOURCE_URL}
-                GIT_TAG
-                ${ARROW_VERSION}
-                GIT_SHALLOW
-                TRUE
-                SOURCE_SUBDIR
-                cpp
-                ${EP_LOG_OPTIONS}
-                CMAKE_ARGS
-                ${ARROW_CMAKE_ARGS}
-                BUILD_COMMAND
-                ""
-                INSTALL_COMMAND
-                ${MAKE} ${MAKE_BUILD_ARGS} install
-                BUILD_BYPRODUCTS
-                "${ARROW_STATIC_LIB}"
-                )
-    endif ()
+    externalproject_add(arrow_ep
+            GIT_REPOSITORY
+            ${ARROW_SOURCE_URL}
+            GIT_TAG
+            ${ARROW_VERSION}
+            GIT_SHALLOW
+            TRUE
+            SOURCE_SUBDIR
+            cpp
+            ${EP_LOG_OPTIONS}
+            CMAKE_ARGS
+            ${ARROW_CMAKE_ARGS}
+            BUILD_COMMAND
+            ""
+            INSTALL_COMMAND
+            ${MAKE} ${MAKE_BUILD_ARGS} install
+            BUILD_BYPRODUCTS
+            "${ARROW_STATIC_LIB}"
+            )
 
     file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
     add_library(arrow STATIC IMPORTED)
@@ -409,39 +349,6 @@ macro(build_gtest)
             "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}"
             )
 
-
-    if (USE_JFROG_CACHE STREQUAL "ON")
-        set(GTEST_CACHE_PACKAGE_NAME "googletest_${GTEST_MD5}.tar.gz")
-        set(GTEST_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GTEST_CACHE_PACKAGE_NAME}")
-        set(GTEST_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GTEST_CACHE_PACKAGE_NAME}")
-
-        execute_process(COMMAND wget -q --method HEAD ${GTEST_CACHE_URL} RESULT_VARIABLE return_code)
-        message(STATUS "Check the remote file ${GTEST_CACHE_URL}. return code = ${return_code}")
-        if (NOT return_code EQUAL 0)
-            ExternalProject_Add(googletest_ep
-                    URL
-                    ${GTEST_SOURCE_URL}
-                    BUILD_COMMAND
-                    ${MAKE}
-                    ${MAKE_BUILD_ARGS}
-                    BUILD_BYPRODUCTS
-                    ${GTEST_STATIC_LIB}
-                    ${GTEST_MAIN_STATIC_LIB}
-                    ${GMOCK_STATIC_LIB}
-                    CMAKE_ARGS
-                    ${GTEST_CMAKE_ARGS}
-                    ${EP_LOG_OPTIONS})
-
-            ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL})
-        else ()
-            file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status)
-            list(GET status 0 status_code)
-            message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-            if (status_code EQUAL 0)
-                ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR})
-            endif ()
-        endif ()
-    else ()
         ExternalProject_Add(googletest_ep
                 URL
                 ${GTEST_SOURCE_URL}
@@ -455,7 +362,6 @@ macro(build_gtest)
                 CMAKE_ARGS
                 ${GTEST_CMAKE_ARGS}
                 ${EP_LOG_OPTIONS})
-    endif ()
 
     # The include directory must exist before it is referenced by a target.
     file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}")
@@ -564,86 +470,40 @@ macro(build_faiss)
         set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda)
     endif ()
 
-    if (USE_JFROG_CACHE STREQUAL "ON")
-        string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}")
-        if (KNOWHERE_GPU_VERSION)
-            set(FAISS_COMPUTE_TYPE "gpu")
-        else ()
-            set(FAISS_COMPUTE_TYPE "cpu")
-        endif ()
-        if (FAISS_WITH_MKL)
-            set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMPUTE_TYPE}_mkl_${FAISS_COMBINE_MD5}.tar.gz")
-        else ()
-            set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMPUTE_TYPE}_openblas_${FAISS_COMBINE_MD5}.tar.gz")
-        endif ()
-        set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}")
-        set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}")
-
-        execute_process(COMMAND wget -q --method HEAD ${FAISS_CACHE_URL} RESULT_VARIABLE return_code)
-        message(STATUS "Check the remote file ${FAISS_CACHE_URL}. return code = ${return_code}")
-        if (NOT return_code EQUAL 0)
-            externalproject_add(faiss_ep
-                    URL
-                    ${FAISS_SOURCE_URL}
-                    ${EP_LOG_OPTIONS}
-                    CONFIGURE_COMMAND
-                    "./configure"
-                    ${FAISS_CONFIGURE_ARGS}
-                    BUILD_COMMAND
-                    ${MAKE} ${MAKE_BUILD_ARGS} all
-                    BUILD_IN_SOURCE
-                    1
-                    INSTALL_COMMAND
-                    ${MAKE} install
-                    BUILD_BYPRODUCTS
-                    ${FAISS_STATIC_LIB})
-
-            ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL})
-        else ()
-            file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status)
-            list(GET status 0 status_code)
-            message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-            if (status_code EQUAL 0)
-                ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR})
-            endif ()
-        endif ()
+    if (CUSTOMIZATION)
+        externalproject_add(faiss_ep
+                DOWNLOAD_COMMAND
+                ""
+                SOURCE_DIR
+                ${FAISS_SOURCE_DIR}
+                ${EP_LOG_OPTIONS}
+                CONFIGURE_COMMAND
+                "./configure"
+                ${FAISS_CONFIGURE_ARGS}
+                BUILD_COMMAND
+                ${MAKE} ${MAKE_BUILD_ARGS} all
+                BUILD_IN_SOURCE
+                1
+                INSTALL_COMMAND
+                ${MAKE} install
+                BUILD_BYPRODUCTS
+                ${FAISS_STATIC_LIB})
     else ()
-        if (CUSTOMIZATION)
-            externalproject_add(faiss_ep
-                    DOWNLOAD_COMMAND
-                    ""
-                    SOURCE_DIR
-                    ${FAISS_SOURCE_DIR}
-                    ${EP_LOG_OPTIONS}
-                    CONFIGURE_COMMAND
-                    "./configure"
-                    ${FAISS_CONFIGURE_ARGS}
-                    BUILD_COMMAND
-                    ${MAKE} ${MAKE_BUILD_ARGS} all
-                    BUILD_IN_SOURCE
-                    1
-                    INSTALL_COMMAND
-                    ${MAKE} install
-                    BUILD_BYPRODUCTS
-                    ${FAISS_STATIC_LIB})
-        else ()
-            externalproject_add(faiss_ep
-                    URL
-                    ${FAISS_SOURCE_URL}
-                    ${EP_LOG_OPTIONS}
-                    CONFIGURE_COMMAND
-                    "./configure"
-                    ${FAISS_CONFIGURE_ARGS}
-                    BUILD_COMMAND
-                    ${MAKE} ${MAKE_BUILD_ARGS} all
-                    BUILD_IN_SOURCE
-                    1
-                    INSTALL_COMMAND
-                    ${MAKE} install
-                    BUILD_BYPRODUCTS
-                    ${FAISS_STATIC_LIB})
-        endif ()
-
+        externalproject_add(faiss_ep
+                URL
+                ${FAISS_SOURCE_URL}
+                ${EP_LOG_OPTIONS}
+                CONFIGURE_COMMAND
+                "./configure"
+                ${FAISS_CONFIGURE_ARGS}
+                BUILD_COMMAND
+                ${MAKE} ${MAKE_BUILD_ARGS} all
+                BUILD_IN_SOURCE
+                1
+                INSTALL_COMMAND
+                ${MAKE} install
+                BUILD_BYPRODUCTS
+                ${FAISS_STATIC_LIB})
     endif ()
 
     file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")

core/src/index/thirdparty/faiss/ci/jenkins/step/build.groovy

-timeout(time: 60, unit: 'MINUTES') {
-    dir ("ci/scripts") {
-        if ("${env.BINRARY_VERSION}" == "gpu") {
-            if ("${params.IS_ORIGIN_FAISS}" == "False") {
-                sh "./build.sh -o ${env.FAISS_ROOT_PATH} -i -g"
-            } else {
-                sh "wget https://github.com/facebookresearch/faiss/archive/v${env.NATIVE_FAISS_VERSION}.tar.gz && \
-                        tar zxvf v${env.NATIVE_FAISS_VERSION}.tar.gz"
-                sh "./build.sh -o ${env.FAISS_ROOT_PATH} -s ./faiss-${env.NATIVE_FAISS_VERSION} -i -g"
-            }
-        } else {
-            sh "wget https://github.com/facebookresearch/faiss/archive/v${env.NATIVE_FAISS_VERSION}.tar.gz && \
-                    tar zxvf v${env.NATIVE_FAISS_VERSION}.tar.gz"
-            sh "./build.sh -o ${env.FAISS_ROOT_PATH} -s ./faiss-${env.NATIVE_FAISS_VERSION} -i"
-        }
-    }
-
-    dir ("milvus") {
-        dir ("ci/scripts") {
-            withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-                def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true)
-                if ("${env.BINRARY_VERSION}" == "gpu") {
-                    if ("${params.IS_ORIGIN_FAISS}" == "False") {
-                        sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -f ${env.FAISS_ROOT_PATH} -l -m -g -x -u -c"
-                    } else {
-                        sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -f ${env.FAISS_ROOT_PATH} -l -m -g -u -c"
-                    }
-                } else {
-                    sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -f ${env.FAISS_ROOT_PATH} -l -m -u -c"
-                }
-                sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}"
-            }
-        }
-    }
-}

core/src/index/thirdparty/faiss/ci/jenkins/step/coverage.groovy

-timeout(time: 30, unit: 'MINUTES') {
-    dir ("milvus/ci/scripts") {
-        sh "./coverage.sh -o /opt/milvus -u root -p 123456 -t \$POD_IP"
-    }
-}
\ No newline at end of file

core/src/index/thirdparty/faiss/ci/scripts/build.sh

-#!/bin/bash
-
-set -e
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
-  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-  SOURCE="$(readlink "$SOURCE")"
-  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
-FAISS_SOURCE_DIR="${SCRIPTS_DIR}/../.."
-
-FAISS_WITH_MKL="False"
-FAISS_GPU_VERSION="False"
-FAISS_COMMON_CONFIGURE_ARGS="CXXFLAGS=\"-mavx2 -mf16c\" --without-python"
-FAISS_CONFIGURE_ARGS="${FAISS_COMMON_CONFIGURE_ARGS}"
-CUDA_TOOLKIT_ROOT_DIR="/usr/local/cuda"
-FAISS_CUDA_ARCH="-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75"
-MKL_ROOT_DIR="/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl"
-
-while getopts "o:s:m:b:l:c:a:igh" arg
-do
-        case $arg in
-             o)
-                FAISS_INSTALL_PREFIX=$OPTARG
-                ;;
-             s)
-                FAISS_SOURCE_DIR=$OPTARG
-                ;;
-             m)
-                MKL_ROOT_DIR=$OPTARG
-                ;;
-             b)
-                OPENBLAS_PREFIX=$OPTARG
-                ;;
-             l)
-                LAPACK_PREFIX=$OPTARG
-                ;;
-             c)
-                CUDA_TOOLKIT_ROOT_DIR=$OPTARG
-                ;;
-             a)
-                FAISS_CUDA_ARCH=$OPTARG
-                ;;
-             i)
-                FAISS_WITH_MKL="True"
-                ;;
-             g)
-                FAISS_GPU_VERSION="True"
-                ;;
-             h) # help
-                echo "
-
-parameter:
--o: faiss install prefix path
--s: faiss source directory
--m: mkl root directory
--b: openblas install prefix path
--l: lapack install prefix path
--c: CUDA toolkit root directory
--a: faiss CUDA compute architecture
--i: faiss with mkl
--g: faiss gpu version
--h: help
-
-usage:
-./build.sh -o \${FAISS_INSTALL_PREFIX} -s \${FAISS_SOURCE_DIR} -m \${MKL_ROOT_DIR} -b \${OPENBLAS_PREFIX} -l \${LAPACK_PREFIX} -c \${CUDA_TOOLKIT_ROOT_DIR} -a \${FAISS_CUDA_ARCH} [-i] [-g] [-h]
-                "
-                exit 0
-                ;;
-             ?)
-                echo "ERROR! unknown argument"
-        exit 1
-        ;;
-        esac
-done
-
-if [[ -n "${FAISS_INSTALL_PREFIX}" ]];then
-        FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} --prefix=${FAISS_INSTALL_PREFIX}"
-fi
-
-if [[ "${FAISS_GPU_VERSION}" == "True" ]];then
-    if [[ ! -n "${FAISS_CUDA_ARCH}" ]];then
-        echo "FAISS_CUDA_ARCH: \"${FAISS_CUDA_ARCH}\" is empty!"
-        exit 1
-    fi
-    if [[ ! -d "${CUDA_TOOLKIT_ROOT_DIR}" ]];then
-        echo "CUDA_TOOLKIT_ROOT_DIR: \"${CUDA_TOOLKIT_ROOT_DIR}\" directory doesn't exist!"
-        exit 1
-    fi
-    FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} --with-cuda=${CUDA_TOOLKIT_ROOT_DIR} --with-cuda-arch='${FAISS_CUDA_ARCH}'"
-else
-    FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} --without-cuda"
-fi
-
-if [[ "${FAISS_WITH_MKL}" == "True" ]];then
-    if [[ ! -d "${MKL_ROOT_DIR}" ]];then
-        echo "MKL_ROOT_DIR: \"${MKL_ROOT_DIR}\" directory doesn't exist!"
-        exit 1
-    fi
-    FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} CPPFLAGS='-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_ROOT_DIR}/include' LDFLAGS='-L${MKL_ROOT_DIR}/lib/intel64'"
-else
-    if [[ -n "${LAPACK_PREFIX}" ]];then
-        if [[ ! -d "${LAPACK_PREFIX}" ]];then
-            echo "LAPACK_PREFIX: \"${LAPACK_PREFIX}\" directory doesn't exist!"
-            exit 1
-        fi
-        FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} LDFLAGS=-L${LAPACK_PREFIX}/lib"
-    fi
-    if [[ -n "${OPENBLAS_PREFIX}" ]];then
-        if [[ ! -d "${OPENBLAS_PREFIX}" ]];then
-            echo "OPENBLAS_PREFIX: \"${OPENBLAS_PREFIX}\" directory doesn't exist!"
-            exit 1
-        fi
-        FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} LDFLAGS=-L${OPENBLAS_PREFIX}/lib"
-    fi
-fi
-
-cd ${FAISS_SOURCE_DIR}
-
-sh -c "./configure ${FAISS_CONFIGURE_ARGS}"
-
-# compile and build
-make -j8 || exit 1
-make install || exit 1

sdk/cmake/ThirdPartyPackages.cmake

@@ -215,67 +215,26 @@ macro(build_grpc)
     set(GRPC_PROTOBUF_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(GRPC_PROTOC_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}")
 
-    if (USE_JFROG_CACHE STREQUAL "ON")
-        set(GRPC_CACHE_PACKAGE_NAME "grpc_${GRPC_MD5}.tar.gz")
-        set(GRPC_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GRPC_CACHE_PACKAGE_NAME}")
-        set(GRPC_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GRPC_CACHE_PACKAGE_NAME}")
-
-        execute_process(COMMAND wget -q --method HEAD ${GRPC_CACHE_URL} RESULT_VARIABLE return_code)
-        message(STATUS "Check the remote file ${GRPC_CACHE_URL}. return code = ${return_code}")
-        if (NOT return_code EQUAL 0)
-            externalproject_add(grpc_ep
-                    URL
-                    ${GRPC_SOURCE_URL}
-                    ${EP_LOG_OPTIONS}
-                    CONFIGURE_COMMAND
-                    ""
-                    BUILD_IN_SOURCE
-                    1
-                    BUILD_COMMAND
-                    ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
-                    INSTALL_COMMAND
-                    ${MAKE} install prefix=${GRPC_PREFIX}
-                    BUILD_BYPRODUCTS
-                    ${GRPC_STATIC_LIB}
-                    ${GRPC++_STATIC_LIB}
-                    ${GRPCPP_CHANNELZ_STATIC_LIB}
-                    ${GRPC_PROTOBUF_STATIC_LIB}
-                    ${GRPC_PROTOC_STATIC_LIB})
-
-            ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
-
-            ExternalProject_Create_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GRPC_CACHE_URL})
-        else ()
-            file(DOWNLOAD ${GRPC_CACHE_URL} ${GRPC_CACHE_PACKAGE_PATH} STATUS status)
-            list(GET status 0 status_code)
-            message(STATUS "DOWNLOADING FROM ${GRPC_CACHE_URL} TO ${GRPC_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-            if (status_code EQUAL 0)
-                ExternalProject_Use_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
-            endif ()
-        endif ()
-    else ()
-        externalproject_add(grpc_ep
-                URL
-                ${GRPC_SOURCE_URL}
-                ${EP_LOG_OPTIONS}
-                CONFIGURE_COMMAND
-                ""
-                BUILD_IN_SOURCE
-                1
-                BUILD_COMMAND
-                ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
-                INSTALL_COMMAND
-                ${MAKE} install prefix=${GRPC_PREFIX}
-                BUILD_BYPRODUCTS
-                ${GRPC_STATIC_LIB}
-                ${GRPC++_STATIC_LIB}
-                ${GRPCPP_CHANNELZ_STATIC_LIB}
-                ${GRPC_PROTOBUF_STATIC_LIB}
-                ${GRPC_PROTOC_STATIC_LIB})
-
-        ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
-
-    endif ()
+    externalproject_add(grpc_ep
+            URL
+            ${GRPC_SOURCE_URL}
+            ${EP_LOG_OPTIONS}
+            CONFIGURE_COMMAND
+            ""
+            BUILD_IN_SOURCE
+            1
+            BUILD_COMMAND
+            ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
+            INSTALL_COMMAND
+            ${MAKE} install prefix=${GRPC_PREFIX}
+            BUILD_BYPRODUCTS
+            ${GRPC_STATIC_LIB}
+            ${GRPC++_STATIC_LIB}
+            ${GRPCPP_CHANNELZ_STATIC_LIB}
+            ${GRPC_PROTOBUF_STATIC_LIB}
+            ${GRPC_PROTOC_STATIC_LIB})
+
+    ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
 
     file(MAKE_DIRECTORY "${GRPC_INCLUDE_DIR}")
 
@@ -338,48 +297,17 @@ macro(build_zlib)
     set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}"
             -DBUILD_SHARED_LIBS=OFF)
 
-    if (USE_JFROG_CACHE STREQUAL "ON")
-        set(ZLIB_CACHE_PACKAGE_NAME "zlib_${ZLIB_MD5}.tar.gz")
-        set(ZLIB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ZLIB_CACHE_PACKAGE_NAME}")
-        set(ZLIB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ZLIB_CACHE_PACKAGE_NAME}")
-
-        execute_process(COMMAND wget -q --method HEAD ${ZLIB_CACHE_URL} RESULT_VARIABLE return_code)
-        message(STATUS "Check the remote file ${ZLIB_CACHE_URL}. return code = ${return_code}")
-        if (NOT return_code EQUAL 0)
-            externalproject_add(zlib_ep
-                    URL
-                    ${ZLIB_SOURCE_URL}
-                    ${EP_LOG_OPTIONS}
-                    BUILD_COMMAND
-                    ${MAKE}
-                    ${MAKE_BUILD_ARGS}
-                    BUILD_BYPRODUCTS
-                    "${ZLIB_STATIC_LIB}"
-                    CMAKE_ARGS
-                    ${ZLIB_CMAKE_ARGS})
-
-            ExternalProject_Create_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ZLIB_CACHE_URL})
-        else ()
-            file(DOWNLOAD ${ZLIB_CACHE_URL} ${ZLIB_CACHE_PACKAGE_PATH} STATUS status)
-            list(GET status 0 status_code)
-            message(STATUS "DOWNLOADING FROM ${ZLIB_CACHE_URL} TO ${ZLIB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-            if (status_code EQUAL 0)
-                ExternalProject_Use_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
-            endif ()
-        endif ()
-    else ()
-        externalproject_add(zlib_ep
-                URL
-                ${ZLIB_SOURCE_URL}
-                ${EP_LOG_OPTIONS}
-                BUILD_COMMAND
-                ${MAKE}
-                ${MAKE_BUILD_ARGS}
-                BUILD_BYPRODUCTS
-                "${ZLIB_STATIC_LIB}"
-                CMAKE_ARGS
-                ${ZLIB_CMAKE_ARGS})
-    endif ()
+    externalproject_add(zlib_ep
+            URL
+            ${ZLIB_SOURCE_URL}
+            ${EP_LOG_OPTIONS}
+            BUILD_COMMAND
+            ${MAKE}
+            ${MAKE_BUILD_ARGS}
+            BUILD_BYPRODUCTS
+            "${ZLIB_STATIC_LIB}"
+            CMAKE_ARGS
+            ${ZLIB_CMAKE_ARGS})
 
     file(MAKE_DIRECTORY "${ZLIB_INCLUDE_DIR}")
     add_library(zlib STATIC IMPORTED)