Merge remote-tracking branch 'source/0.6.0' into 0.6.0

CHANGELOG.md

@@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#412 - Message returned is confused when partition created with null partition name
 - \#416 - Drop the same partition success repeatally
 - \#440 - Query API in customization still uses old version
+- \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version
 - \#458 - Index data is not compatible between 0.5 and 0.6
 
 ## Feature

ci/jenkins/step/package.groovy

@@ -2,6 +2,7 @@ timeout(time: 5, unit: 'MINUTES') {
     dir ("ci/jenkins/scripts") {
         sh "pip3 install -r requirements.txt"
         sh "./yaml_processor.py merge -f /opt/milvus/conf/server_config.yaml -m ../yaml/update_server_config.yaml -i && rm /opt/milvus/conf/server_config.yaml.bak"
+        sh "sed -i 's/\/tmp\/milvus/\/opt\/milvus/g' /opt/milvus/conf/log_config.conf"
     }
     sh "tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz -C /opt/ milvus"
     withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {

ci/jenkinsfile/cleanup_staging.groovy

-try {
-    def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
-    if (!result) {
-        sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
-    }
-} catch (exc) {
-    def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
-    if (!result) {
-        sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
-    }
-    throw exc
-}
-

ci/jenkinsfile/cluster_cleanup_dev.groovy

-try {
-    def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true
-    if (!result) {
-        sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster"
-    }
-} catch (exc) {
-    def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true
-    if (!result) {
-        sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster"
-    }
-    throw exc
-}
-

ci/jenkinsfile/cluster_deploy2dev.groovy

-try {
-    sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
-    sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus'
-    sh 'helm repo update'
-    dir ("milvus-helm") {
-        checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-        dir ("milvus/milvus-cluster") {
-            sh "helm install --wait --timeout 300 --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.5.0 . "
-        }
-    }
-    /*
-    timeout(time: 2, unit: 'MINUTES') {
-        waitUntil {
-            def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true
-            return !result
-        }
-    }
-    */
-} catch (exc) {
-    echo 'Helm running failed!'
-    sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster"
-    throw exc
-}
-

ci/jenkinsfile/cluster_dev_test.groovy

-timeout(time: 25, unit: 'MINUTES') {
-    try {
-        dir ("${PROJECT_NAME}_test") {
-            checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-            sh 'python3 -m pip install -r requirements_cluster.txt'
-            sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local"
-        }
-    } catch (exc) {
-        echo 'Milvus Cluster Test Failed !'
-        throw exc
-    }
-}

ci/jenkinsfile/deploy2dev.groovy

-try {
-    sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
-    sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus'
-    sh 'helm repo update'
-    dir ("milvus-helm") {
-        checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-        dir ("milvus/milvus-gpu") {
-            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ."
-        }
-    }
-} catch (exc) {
-    echo 'Helm running failed!'
-    sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
-    throw exc
-}
-

ci/jenkinsfile/deploy2staging.groovy

-try {
-    sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
-    sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus'
-    sh 'helm repo update'
-    dir ("milvus-helm") {
-        checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-        dir ("milvus/milvus-gpu") {
-            sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ."
-        }
-    }
-} catch (exc) {
-    echo 'Helm running failed!'
-    sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
-    throw exc
-}
-

ci/jenkinsfile/dev_test.groovy

-timeout(time: 30, unit: 'MINUTES') {
-    try {
-        dir ("${PROJECT_NAME}_test") {
-            checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-            sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
-            sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal=true"
-        }
-        // mysql database backend test
-        load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
-
-        if (!fileExists('milvus-helm')) {
-            dir ("milvus-helm") {
-                checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-            }
-        }
-        dir ("milvus-helm") {
-            dir ("milvus/milvus-gpu") {
-                sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ."
-            }
-        }
-        dir ("${PROJECT_NAME}_test") {
-            sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal=true"
-        }
-    } catch (exc) {
-        echo 'Milvus Test Failed !'
-        throw exc
-    }
-}

ci/jenkinsfile/dev_test_all.groovy

-timeout(time: 60, unit: 'MINUTES') {
-    try {
-        dir ("${PROJECT_NAME}_test") {
-            checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-            sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
-            sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal=true"
-        }
-
-        // mysql database backend test
-        load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
-
-        if (!fileExists('milvus-helm')) {
-            dir ("milvus-helm") {
-                checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-            }
-        }
-        dir ("milvus-helm") {
-            dir ("milvus/milvus-gpu") {
-                sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ."
-            }
-        }
-        dir ("${PROJECT_NAME}_test") {
-            sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal=true"
-        }
-    } catch (exc) {
-        echo 'Milvus Test Failed !'
-        throw exc
-    }
-}

ci/jenkinsfile/milvus_build.groovy

-container('milvus-build-env') {
-    timeout(time: 120, unit: 'MINUTES') {
-        gitlabCommitStatus(name: 'Build Engine') {
-            dir ("milvus_engine") {
-                try {
-                    checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-
-                    dir ("core") {
-                        sh "git config --global user.email \"test@zilliz.com\""
-                        sh "git config --global user.name \"test\""
-                        withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-                            sh "./build.sh -l"
-                            sh "rm -rf cmake_build"
-                            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \
-                            && export JFROG_USER_NAME='${USERNAME}' \
-                            && export JFROG_PASSWORD='${PASSWORD}' \
-                            && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \
-                            && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u -c"
-
-                            sh "./coverage.sh -u root -p 123456 -t \$POD_IP"
-                        }
-                    }
-                } catch (exc) {
-                    updateGitlabCommitStatus name: 'Build Engine', state: 'failed'
-                    throw exc
-                }
-            }
-        }
-    }
-}

ci/jenkinsfile/milvus_build_no_ut.groovy

-container('milvus-build-env') {
-    timeout(time: 120, unit: 'MINUTES') {
-        gitlabCommitStatus(name: 'Build Engine') {
-            dir ("milvus_engine") {
-                try {
-                    checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-
-                    dir ("core") {
-                        sh "git config --global user.email \"test@zilliz.com\""
-                        sh "git config --global user.name \"test\""
-                        withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-                            sh "./build.sh -l"
-                            sh "rm -rf cmake_build"
-                            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \
-                            && export JFROG_USER_NAME='${USERNAME}' \
-                            && export JFROG_PASSWORD='${PASSWORD}' \
-                            && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \
-                            && ./build.sh -t ${params.BUILD_TYPE} -j -d /opt/milvus"
-                        }
-                    }
-                } catch (exc) {
-                    updateGitlabCommitStatus name: 'Build Engine', state: 'failed'
-                    throw exc
-                }
-            }
-        }
-    }
-}

ci/jenkinsfile/nightly_publish_docker.groovy

-container('publish-docker') {
-    timeout(time: 15, unit: 'MINUTES') {
-        gitlabCommitStatus(name: 'Publish Engine Docker') {
-            try {
-                dir ("${PROJECT_NAME}_build") {
-                    checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-                    dir ("docker/deploy/ubuntu16.04/free_version") {
-                        sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz"
-                        sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz"
-                        try {
-                            def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}")
-                            docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_PUBLISH_USER}") {
-                                customImage.push()
-                            }
-                            docker.withRegistry('https://zilliz.azurecr.cn', "${params.AZURE_DOCKER_PUBLISH_USER}") {
-                                customImage.push()
-                            }
-                            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
-                                updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'success'
-                                echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}"
-                            }
-                        } catch (exc) {
-                            updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled'
-                            throw exc
-                        } finally {
-                            sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}"
-                        }
-                    }
-                }
-            } catch (exc) {
-                updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed'
-                echo 'Publish docker failed!'
-                throw exc
-            }
-        }
-    }
-}
-

ci/jenkinsfile/packaged_milvus.groovy

-container('milvus-build-env') {
-    timeout(time: 5, unit: 'MINUTES') {
-        dir ("milvus_engine") {
-            dir ("core") {
-                gitlabCommitStatus(name: 'Packaged Engine') {
-                    if (fileExists('milvus')) {
-                        try {
-                            sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus"
-                            def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
-                            fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage')
-                            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
-                                echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\""
-                            }
-                        } catch (exc) {
-                            updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed'
-                            throw exc
-                        }
-                    } else {
-                        updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed'
-                        error("Milvus binary directory don't exists!")
-                    }
-                }
-
-                gitlabCommitStatus(name: 'Packaged Engine lcov') {
-                    if (fileExists('lcov_out')) {
-                        try {
-                            def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
-                            fileTransfer.FileTransfer("lcov_out/", "${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}", 'nas storage')
-                            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
-                                echo "Milvus lcov out Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}/lcov_out/\""
-                            }
-                        } catch (exc) {
-                            updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed'
-                            throw exc
-                        }
-                    } else {
-                        updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed'
-                        error("Milvus lcov out directory don't exists!")
-                    }
-                }
-            }
-        }
-    }
-}

ci/jenkinsfile/packaged_milvus_no_ut.groovy

-container('milvus-build-env') {
-    timeout(time: 5, unit: 'MINUTES') {
-        dir ("milvus_engine") {
-            dir ("core") {
-                gitlabCommitStatus(name: 'Packaged Engine') {
-                    if (fileExists('milvus')) {
-                        try {
-                            sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus"
-                            def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
-                            fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage')
-                            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
-                                echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\""
-                            }
-                        } catch (exc) {
-                            updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed'
-                            throw exc
-                        }
-                    } else {
-                        updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed'
-                        error("Milvus binary directory don't exists!")
-                    }
-                }
-            }
-        }
-    }
-}

ci/jenkinsfile/publish_docker.groovy

-container('publish-docker') {
-    timeout(time: 15, unit: 'MINUTES') {
-        gitlabCommitStatus(name: 'Publish Engine Docker') {
-            try {
-                dir ("${PROJECT_NAME}_build") {
-                    checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-                    dir ("docker/deploy/ubuntu16.04/free_version") {
-                        sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz"
-                        sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz"
-                        try {
-                            def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}")
-                            docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_PUBLISH_USER}") {
-                                customImage.push()
-                            }
-                            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
-                                updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'success'
-                                echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}"
-                            }
-                        } catch (exc) {
-                            updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled'
-                            throw exc
-                        } finally {
-                            sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}"
-                        }
-                    }
-                }
-            } catch (exc) {
-                updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed'
-                echo 'Publish docker failed!'
-                throw exc
-            }
-        }
-    }
-}
-

ci/jenkinsfile/staging_test.groovy

-timeout(time: 40, unit: 'MINUTES') {
-    try {
-        dir ("${PROJECT_NAME}_test") {
-            checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-            sh 'python3 -m pip install -r requirements.txt'
-            def service_ip = sh (script: "kubectl get svc --namespace milvus-1 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim()
-            sh "pytest . --alluredir=\"test_out/staging/single/sqlite\" --ip ${service_ip}"
-        }
-
-        // mysql database backend test
-        load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
-
-        if (!fileExists('milvus-helm')) {
-            dir ("milvus-helm") {
-                checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-            }
-        }
-        dir ("milvus-helm") {
-            dir ("milvus/milvus-gpu") {
-                sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ."
-            }
-        }
-        dir ("${PROJECT_NAME}_test") {
-            def service_ip = sh (script: "kubectl get svc --namespace milvus-2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim()
-            sh "pytest . --alluredir=\"test_out/staging/single/mysql\" --ip ${service_ip}"
-        }
-    } catch (exc) {
-        echo 'Milvus Test Failed !'
-        throw exc
-    }
-}

ci/jenkinsfile/upload_dev_cluster_test_out.groovy

-timeout(time: 5, unit: 'MINUTES') {
-    dir ("${PROJECT_NAME}_test") {
-        if (fileExists('cluster_test_out')) {
-            def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
-            fileTransfer.FileTransfer("cluster_test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
-            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
-                echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
-            }
-        } else {
-            error("Milvus Dev Test Out directory don't exists!")
-        }
-    }
-}
-

ci/jenkinsfile/upload_dev_test_out.groovy

-timeout(time: 5, unit: 'MINUTES') {
-    dir ("${PROJECT_NAME}_test") {
-        if (fileExists('test_out/dev')) {
-            def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
-            fileTransfer.FileTransfer("test_out/dev/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
-            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
-                echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
-            }
-        } else {
-            error("Milvus Dev Test Out directory don't exists!")
-        }
-    }
-}

ci/jenkinsfile/upload_staging_test_out.groovy

-timeout(time: 5, unit: 'MINUTES') {
-    dir ("${PROJECT_NAME}_test") {
-        if (fileExists('test_out/staging')) {
-            def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
-            fileTransfer.FileTransfer("test_out/staging/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
-            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
-                echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
-            }
-        } else {
-            error("Milvus Dev Test Out directory don't exists!")
-        }
-    }
-}

ci/main_jenkinsfile

-pipeline {
-    agent none
-
-    options {
-        timestamps()
-    }
-
-    environment {
-        PROJECT_NAME = "milvus"
-        LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase()
-        SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}"
-        GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}"
-        SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}"
-        DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}"
-    }
-
-    stages {
-        stage("Ubuntu 16.04") {
-            environment {
-                PACKAGE_VERSION = VersionNumber([
-                    versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}'
-                ]);
-
-                DOCKER_VERSION = VersionNumber([
-                    versionNumberString : '${DOCKER_VERSION_STR}'
-                ]);
-            }
-
-            stages {
-                stage("Run Build") {
-                    agent {
-                        kubernetes {
-                            cloud 'build-kubernetes'
-                            label 'build'
-                            defaultContainer 'jnlp'
-                            yaml """
-apiVersion: v1
-kind: Pod
-metadata:
-  name: milvus-build-env
-  labels:
-    app: milvus
-    componet: build-env
-spec:
-  containers:
-  - name: milvus-build-env
-    image: registry.zilliz.com/milvus/milvus-build-env:v0.13
-    env:
-    - name: POD_IP
-      valueFrom:
-        fieldRef:
-          fieldPath: status.podIP
-    command:
-    - cat
-    tty: true
-    resources:
-      limits:
-        memory: "28Gi"
-        cpu: "10.0"
-        nvidia.com/gpu: 1
-      requests:
-        memory: "14Gi"
-        cpu: "5.0"
-  - name: milvus-mysql
-    image: mysql:5.6
-    env:
-    - name: MYSQL_ROOT_PASSWORD
-      value: 123456
-    ports:
-    - containerPort: 3306
-      name: mysql
-"""
-                        }
-                    }
-                    stages {
-                        stage('Build') {
-                            steps {
-                                gitlabCommitStatus(name: 'Build') {
-                                    script {
-                                        load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy"
-                                        load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                    post {
-                        aborted {
-                            script {
-                                updateGitlabCommitStatus name: 'Build', state: 'canceled'
-                                echo "Milvus Build aborted !"
-                            }
-                        }
-
-                        failure {
-                            script {
-                                updateGitlabCommitStatus name: 'Build', state: 'failed'
-                                echo "Milvus Build failure !"
-                            }
-                        }
-                    }
-                }
-
-                stage("Publish docker and helm") {
-                    agent {
-                        kubernetes {
-                            label 'publish'
-                            defaultContainer 'jnlp'
-                            yaml """
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: publish
-    componet: docker
-spec:
-  containers:
-  - name: publish-docker
-    image: registry.zilliz.com/library/zilliz_docker:v1.0.0
-    securityContext:
-      privileged: true
-    command:
-    - cat
-    tty: true
-    volumeMounts:
-    - name: docker-sock
-      mountPath: /var/run/docker.sock
-  volumes:
-  - name: docker-sock
-    hostPath:
-      path: /var/run/docker.sock
-"""
-                        }
-                    }
-                    stages {
-                        stage('Publish Docker') {
-                            steps {
-                                gitlabCommitStatus(name: 'Publish Docker') {
-                                    script {
-                                        load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                    post {
-                        aborted {
-                            script {
-                                updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled'
-                                echo "Milvus Publish Docker aborted !"
-                            }
-                        }
-
-                        failure {
-                            script {
-                                updateGitlabCommitStatus name: 'Publish Docker', state: 'failed'
-                                echo "Milvus Publish Docker failure !"
-                            }
-                        }
-                    }
-                }
-
-                stage("Deploy to Development") {
-                    parallel {
-                        stage("Single Node") {
-                            agent {
-                                kubernetes {
-                                    label 'dev-test'
-                                    defaultContainer 'jnlp'
-                                    yaml """
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: milvus
-    componet: test
-spec:
-  containers:
-  - name: milvus-testframework
-    image: registry.zilliz.com/milvus/milvus-test:v0.2
-    command:
-    - cat
-    tty: true
-    volumeMounts:
-    - name: kubeconf
-      mountPath: /root/.kube/
-      readOnly: true
-  volumes:
-  - name: kubeconf
-    secret:
-      secretName: test-cluster-config
-"""
-                                }
-                            }
-
-                            stages {
-                                stage("Deploy to Dev") {
-                                    steps {
-                                        gitlabCommitStatus(name: 'Deloy to Dev') {
-                                            container('milvus-testframework') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                                stage("Dev Test") {
-                                    steps {
-                                        gitlabCommitStatus(name: 'Deloy Test') {
-                                            container('milvus-testframework') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy"
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                                stage ("Cleanup Dev") {
-                                    steps {
-                                        gitlabCommitStatus(name: 'Cleanup Dev') {
-                                            container('milvus-testframework') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                            }
-                            post {
-                                always {
-                                    container('milvus-testframework') {
-                                        script {
-                                            load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
-                                        }
-                                    }
-                                }
-                                success {
-                                    script {
-                                        echo "Milvus Single Node CI/CD success !"
-                                    }
-                                }
-                                aborted {
-                                    script {
-                                        echo "Milvus Single Node CI/CD aborted !"
-                                    }
-                                }
-                                failure {
-                                    script {
-                                        echo "Milvus Single Node CI/CD failure !"
-                                    }
-                                }
-                            }
-                        }
-
-//                         stage("Cluster") {
-//                             agent {
-//                                 kubernetes {
-//                                     label 'dev-test'
-//                                     defaultContainer 'jnlp'
-//                                     yaml """
-// apiVersion: v1
-// kind: Pod
-// metadata:
-//   labels:
-//     app: milvus
-//     componet: test
-// spec:
-//   containers:
-//   - name: milvus-testframework
-//     image: registry.zilliz.com/milvus/milvus-test:v0.2
-//     command:
-//     - cat
-//     tty: true
-//     volumeMounts:
-//     - name: kubeconf
-//       mountPath: /root/.kube/
-//       readOnly: true
-//   volumes:
-//   - name: kubeconf
-//     secret:
-//       secretName: test-cluster-config
-// """
-//                                 }
-//                             }
-//                             stages {
-//                                 stage("Deploy to Dev") {
-//                                     steps {
-//                                         gitlabCommitStatus(name: 'Deloy to Dev') {
-//                                             container('milvus-testframework') {
-//                                                 script {
-//                                                     load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy"
-//                                                 }
-//                                             }
-//                                         }
-//                                     }
-//                                 }
-//                                 stage("Dev Test") {
-//                                     steps {
-//                                         gitlabCommitStatus(name: 'Deloy Test') {
-//                                             container('milvus-testframework') {
-//                                                 script {
-//                                                     load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy"
-//                                                     load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy"
-//                                                 }
-//                                             }
-//                                         }
-//                                     }
-//                                 }
-//                                 stage ("Cleanup Dev") {
-//                                     steps {
-//                                         gitlabCommitStatus(name: 'Cleanup Dev') {
-//                                             container('milvus-testframework') {
-//                                                 script {
-//                                                     load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy"
-//                                                 }
-//                                             }
-//                                         }
-//                                     }
-//                                 }
-//                             }
-//                             post {
-//                                 always {
-//                                     container('milvus-testframework') {
-//                                         script {
-//                                             load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy"
-//                                         }
-//                                     }
-//                                 }
-//                                 success {
-//                                     script {
-//                                         echo "Milvus Cluster CI/CD success !"
-//                                     }
-//                                 }
-//                                 aborted {
-//                                     script {
-//                                         echo "Milvus Cluster CI/CD aborted !"
-//                                     }
-//                                 }
-//                                 failure {
-//                                     script {
-//                                         echo "Milvus Cluster CI/CD failure !"
-//                                     }
-//                                 }
-//                             }
-//                        }
-                    }
-                }
-            }
-        }
-    }
-
-    post {
-        always {
-            script {
-                if (env.gitlabAfter != null) {
-                    if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
-                        // Send an email only if the build status has changed from green/unstable to red
-                        emailext subject: '$DEFAULT_SUBJECT',
-                        body: '$DEFAULT_CONTENT',
-                        recipientProviders: [
-                            [$class: 'DevelopersRecipientProvider'],
-                            [$class: 'RequesterRecipientProvider']
-                        ],
-                        replyTo: '$DEFAULT_REPLYTO',
-                        to: '$DEFAULT_RECIPIENTS'
-                    }
-                }
-            }
-        }
-
-        success {
-            script {
-                updateGitlabCommitStatus name: 'CI/CD', state: 'success'
-                echo "Milvus CI/CD success !"
-            }
-        }
-
-        aborted {
-            script {
-                updateGitlabCommitStatus name: 'CI/CD', state: 'canceled'
-                echo "Milvus CI/CD aborted !"
-            }
-        }
-
-        failure {
-            script {
-                updateGitlabCommitStatus name: 'CI/CD', state: 'failed'
-                echo "Milvus CI/CD failure !"
-            }
-        }
-    }
-}
-

ci/main_jenkinsfile_no_ut

-pipeline {
-    agent none
-
-    options {
-        timestamps()
-    }
-
-    environment {
-        PROJECT_NAME = "milvus"
-        LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase()
-        SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}"
-        GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}"
-        SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}"
-        DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}"
-    }
-
-    stages {
-        stage("Ubuntu 16.04") {
-            environment {
-                PACKAGE_VERSION = VersionNumber([
-                    versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}'
-                ]);
-
-                DOCKER_VERSION = VersionNumber([
-                    versionNumberString : '${DOCKER_VERSION_STR}'
-                ]);
-            }
-
-            stages {
-                stage("Run Build") {
-                    agent {
-                        kubernetes {
-                            cloud 'build-kubernetes'
-                            label 'build'
-                            defaultContainer 'jnlp'
-                            yaml """
-apiVersion: v1
-kind: Pod
-metadata:
-  name: milvus-build-env
-  labels:
-    app: milvus
-    componet: build-env
-spec:
-  containers:
-  - name: milvus-build-env
-    image: registry.zilliz.com/milvus/milvus-build-env:v0.13
-    env:
-    - name: POD_IP
-      valueFrom:
-        fieldRef:
-          fieldPath: status.podIP
-    command:
-    - cat
-    tty: true
-    resources:
-      limits:
-        memory: "28Gi"
-        cpu: "10.0"
-        nvidia.com/gpu: 1
-      requests:
-        memory: "14Gi"
-        cpu: "5.0"
-  - name: milvus-mysql
-    image: mysql:5.6
-    env:
-    - name: MYSQL_ROOT_PASSWORD
-      value: 123456
-    ports:
-    - containerPort: 3306
-      name: mysql
-"""
-                        }
-                    }
-                    stages {
-                        stage('Build') {
-                            steps {
-                                gitlabCommitStatus(name: 'Build') {
-                                    script {
-                                        load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build_no_ut.groovy"
-                                        load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus_no_ut.groovy"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                    post {
-                        aborted {
-                            script {
-                                updateGitlabCommitStatus name: 'Build', state: 'canceled'
-                                echo "Milvus Build aborted !"
-                            }
-                        }
-
-                        failure {
-                            script {
-                                updateGitlabCommitStatus name: 'Build', state: 'failed'
-                                echo "Milvus Build failure !"
-                            }
-                        }
-                    }
-                }
-
-                stage("Publish docker and helm") {
-                    agent {
-                        kubernetes {
-                            label 'publish'
-                            defaultContainer 'jnlp'
-                            yaml """
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: publish
-    componet: docker
-spec:
-  containers:
-  - name: publish-docker
-    image: registry.zilliz.com/library/zilliz_docker:v1.0.0
-    securityContext:
-      privileged: true
-    command:
-    - cat
-    tty: true
-    volumeMounts:
-    - name: docker-sock
-      mountPath: /var/run/docker.sock
-  volumes:
-  - name: docker-sock
-    hostPath:
-      path: /var/run/docker.sock
-"""
-                        }
-                    }
-                    stages {
-                        stage('Publish Docker') {
-                            steps {
-                                gitlabCommitStatus(name: 'Publish Docker') {
-                                    script {
-                                        load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                    post {
-                        aborted {
-                            script {
-                                updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled'
-                                echo "Milvus Publish Docker aborted !"
-                            }
-                        }
-
-                        failure {
-                            script {
-                                updateGitlabCommitStatus name: 'Publish Docker', state: 'failed'
-                                echo "Milvus Publish Docker failure !"
-                            }
-                        }
-                    }
-                }
-
-                stage("Deploy to Development") {
-                    parallel {
-                        stage("Single Node") {
-                            agent {
-                                kubernetes {
-                                    label 'dev-test'
-                                    defaultContainer 'jnlp'
-                                    yaml """
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: milvus
-    componet: test
-spec:
-  containers:
-  - name: milvus-testframework
-    image: registry.zilliz.com/milvus/milvus-test:v0.2
-    command:
-    - cat
-    tty: true
-    volumeMounts:
-    - name: kubeconf
-      mountPath: /root/.kube/
-      readOnly: true
-  volumes:
-  - name: kubeconf
-    secret:
-      secretName: test-cluster-config
-"""
-                                }
-                            }
-
-                            stages {
-                                stage("Deploy to Dev") {
-                                    steps {
-                                        gitlabCommitStatus(name: 'Deloy to Dev') {
-                                            container('milvus-testframework') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                                stage("Dev Test") {
-                                    steps {
-                                        gitlabCommitStatus(name: 'Deloy Test') {
-                                            container('milvus-testframework') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy"
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                                stage ("Cleanup Dev") {
-                                    steps {
-                                        gitlabCommitStatus(name: 'Cleanup Dev') {
-                                            container('milvus-testframework') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                            }
-                            post {
-                                always {
-                                    container('milvus-testframework') {
-                                        script {
-                                            load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
-                                        }
-                                    }
-                                }
-                                success {
-                                    script {
-                                        echo "Milvus Single Node CI/CD success !"
-                                    }
-                                }
-                                aborted {
-                                    script {
-                                        echo "Milvus Single Node CI/CD aborted !"
-                                    }
-                                }
-                                failure {
-                                    script {
-                                        echo "Milvus Single Node CI/CD failure !"
-                                    }
-                                }
-                            }
-                        }
-
-//                         stage("Cluster") {
-//                             agent {
-//                                 kubernetes {
-//                                     label 'dev-test'
-//                                     defaultContainer 'jnlp'
-//                                     yaml """
-// apiVersion: v1
-// kind: Pod
-// metadata:
-//   labels:
-//     app: milvus
-//     componet: test
-// spec:
-//   containers:
-//   - name: milvus-testframework
-//     image: registry.zilliz.com/milvus/milvus-test:v0.2
-//     command:
-//     - cat
-//     tty: true
-//     volumeMounts:
-//     - name: kubeconf
-//       mountPath: /root/.kube/
-//       readOnly: true
-//   volumes:
-//   - name: kubeconf
-//     secret:
-//       secretName: test-cluster-config
-// """
-//                                 }
-//                             }
-//                             stages {
-//                                 stage("Deploy to Dev") {
-//                                     steps {
-//                                         gitlabCommitStatus(name: 'Deloy to Dev') {
-//                                             container('milvus-testframework') {
-//                                                 script {
-//                                                     load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy"
-//                                                 }
-//                                             }
-//                                         }
-//                                     }
-//                                 }
-//                                 stage("Dev Test") {
-//                                     steps {
-//                                         gitlabCommitStatus(name: 'Deloy Test') {
-//                                             container('milvus-testframework') {
-//                                                 script {
-//                                                     load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy"
-//                                                     load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy"
-//                                                 }
-//                                             }
-//                                         }
-//                                     }
-//                                 }
-//                                 stage ("Cleanup Dev") {
-//                                     steps {
-//                                         gitlabCommitStatus(name: 'Cleanup Dev') {
-//                                             container('milvus-testframework') {
-//                                                 script {
-//                                                     load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy"
-//                                                 }
-//                                             }
-//                                         }
-//                                     }
-//                                 }
-//                             }
-//                             post {
-//                                 always {
-//                                     container('milvus-testframework') {
-//                                         script {
-//                                             load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy"
-//                                         }
-//                                     }
-//                                 }
-//                                 success {
-//                                     script {
-//                                         echo "Milvus Cluster CI/CD success !"
-//                                     }
-//                                 }
-//                                 aborted {
-//                                     script {
-//                                         echo "Milvus Cluster CI/CD aborted !"
-//                                     }
-//                                 }
-//                                 failure {
-//                                     script {
-//                                         echo "Milvus Cluster CI/CD failure !"
-//                                     }
-//                                 }
-//                             }
-//                         }
-                    }
-                }
-            }
-        }
-    }
-
-    post {
-        always {
-            script {
-                if (env.gitlabAfter != null) {
-                    if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
-                        // Send an email only if the build status has changed from green/unstable to red
-                        emailext subject: '$DEFAULT_SUBJECT',
-                        body: '$DEFAULT_CONTENT',
-                        recipientProviders: [
-                            [$class: 'DevelopersRecipientProvider'],
-                            [$class: 'RequesterRecipientProvider']
-                        ],
-                        replyTo: '$DEFAULT_REPLYTO',
-                        to: '$DEFAULT_RECIPIENTS'
-                    }
-                }
-            }
-        }
-
-        success {
-            script {
-                updateGitlabCommitStatus name: 'CI/CD', state: 'success'
-                echo "Milvus CI/CD success !"
-            }
-        }
-
-        aborted {
-            script {
-                updateGitlabCommitStatus name: 'CI/CD', state: 'canceled'
-                echo "Milvus CI/CD aborted !"
-            }
-        }
-
-        failure {
-            script {
-                updateGitlabCommitStatus name: 'CI/CD', state: 'failed'
-                echo "Milvus CI/CD failure !"
-            }
-        }
-    }
-}
-

ci/nightly_main_jenkinsfile

-pipeline {
-    agent none
-
-    options {
-        timestamps()
-    }
-
-    environment {
-        PROJECT_NAME = "milvus"
-        LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase()
-        SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}"
-        GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}"
-        SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}"
-        DOCKER_VERSION_STR = "${env.gitlabAfter == null ? '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, \"yyyyMMdd\"}' : '${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}'}"
-    }
-
-    stages {
-        stage("Ubuntu 16.04") {
-            environment {
-                PACKAGE_VERSION = VersionNumber([
-                    versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}'
-                ]);
-
-                DOCKER_VERSION = VersionNumber([
-                    versionNumberString : '${DOCKER_VERSION_STR}'
-                ]);
-            }
-
-            stages {
-                stage("Run Build") {
-                    agent {
-                        kubernetes {
-                            cloud 'build-kubernetes'
-                            label 'build'
-                            defaultContainer 'jnlp'
-                            yaml """
-apiVersion: v1
-kind: Pod
-metadata:
-  name: milvus-build-env
-  labels:
-    app: milvus
-    componet: build-env
-spec:
-  containers:
-  - name: milvus-build-env
-    image: registry.zilliz.com/milvus/milvus-build-env:v0.13
-    command:
-    - cat
-    tty: true
-    resources:
-      limits:
-        memory: "28Gi"
-        cpu: "10.0"
-        nvidia.com/gpu: 1
-      requests:
-        memory: "14Gi"
-        cpu: "5.0"
-"""
-                        }
-                    }
-                    stages {
-                        stage('Build') {
-                            steps {
-                                gitlabCommitStatus(name: 'Build') {
-                                    script {
-                                        load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy"
-                                        load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                    post {
-                        aborted {
-                            script {
-                                updateGitlabCommitStatus name: 'Build', state: 'canceled'
-                                echo "Milvus Build aborted !"
-                            }
-                        }
-
-                        failure {
-                            script {
-                                updateGitlabCommitStatus name: 'Build', state: 'failed'
-                                echo "Milvus Build failure !"
-                            }
-                        }
-                    }
-                }
-
-                stage("Publish docker and helm") {
-                    agent {
-                        kubernetes {
-                            label 'publish'
-                            defaultContainer 'jnlp'
-                            yaml """
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: publish
-    componet: docker
-spec:
-  containers:
-  - name: publish-docker
-    image: registry.zilliz.com/library/zilliz_docker:v1.0.0
-    securityContext:
-      privileged: true
-    command:
-    - cat
-    tty: true
-    volumeMounts:
-    - name: docker-sock
-      mountPath: /var/run/docker.sock
-  volumes:
-  - name: docker-sock
-    hostPath:
-      path: /var/run/docker.sock
-"""
-                        }
-                    }
-                    stages {
-                        stage('Publish Docker') {
-                            steps {
-                                gitlabCommitStatus(name: 'Publish Docker') {
-                                    script {
-                                        load "${env.WORKSPACE}/ci/jenkinsfile/nightly_publish_docker.groovy"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                    post {
-                        aborted {
-                            script {
-                                updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled'
-                                echo "Milvus Publish Docker aborted !"
-                            }
-                        }
-
-                        failure {
-                            script {
-                                updateGitlabCommitStatus name: 'Publish Docker', state: 'failed'
-                                echo "Milvus Publish Docker failure !"
-                            }
-                        }
-                    }
-                }
-
-                stage("Deploy to Development") {
-                    parallel {
-                        stage("Single Node") {
-                            agent {
-                                kubernetes {
-                                    label 'dev-test'
-                                    defaultContainer 'jnlp'
-                                    yaml """
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: milvus
-    componet: test
-spec:
-  containers:
-  - name: milvus-testframework
-    image: registry.zilliz.com/milvus/milvus-test:v0.2
-    command:
-    - cat
-    tty: true
-    volumeMounts:
-    - name: kubeconf
-      mountPath: /root/.kube/
-      readOnly: true
-  volumes:
-  - name: kubeconf
-    secret:
-      secretName: test-cluster-config
-"""
-                                }
-                            }
-
-                            stages {
-                                stage("Deploy to Dev") {
-                                    steps {
-                                        gitlabCommitStatus(name: 'Deloy to Dev') {
-                                            container('milvus-testframework') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                                stage("Dev Test") {
-                                    steps {
-                                        gitlabCommitStatus(name: 'Deloy Test') {
-                                            container('milvus-testframework') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/dev_test_all.groovy"
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                                stage ("Cleanup Dev") {
-                                    steps {
-                                        gitlabCommitStatus(name: 'Cleanup Dev') {
-                                            container('milvus-testframework') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                            }
-                            post {
-                                always {
-                                    container('milvus-testframework') {
-                                        script {
-                                            load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
-                                        }
-                                    }
-                                }
-                                success {
-                                    script {
-                                        echo "Milvus Deploy to Dev Single Node CI/CD success !"
-                                    }
-                                }
-                                aborted {
-                                    script {
-                                        echo "Milvus Deploy to Dev Single Node CI/CD aborted !"
-                                    }
-                                }
-                                failure {
-                                    script {
-                                        echo "Milvus Deploy to Dev Single Node CI/CD failure !"
-                                    }
-                                }
-                            }
-                        }
-
-//                         stage("Cluster") {
-//                             agent {
-//                                 kubernetes {
-//                                     label 'dev-test'
-//                                     defaultContainer 'jnlp'
-//                                     yaml """
-// apiVersion: v1
-// kind: Pod
-// metadata:
-//   labels:
-//     app: milvus
-//     componet: test
-// spec:
-//   containers:
-//   - name: milvus-testframework
-//     image: registry.zilliz.com/milvus/milvus-test:v0.2
-//     command:
-//     - cat
-//     tty: true
-//     volumeMounts:
-//     - name: kubeconf
-//       mountPath: /root/.kube/
-//       readOnly: true
-//   volumes:
-//   - name: kubeconf
-//     secret:
-//       secretName: test-cluster-config
-// """
-//                                 }
-//                             }
-//                             stages {
-//                                 stage("Deploy to Dev") {
-//                                     steps {
-//                                         gitlabCommitStatus(name: 'Deloy to Dev') {
-//                                             container('milvus-testframework') {
-//                                                 script {
-//                                                     load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy"
-//                                                 }
-//                                             }
-//                                         }
-//                                     }
-//                                 }
-//                                 stage("Dev Test") {
-//                                     steps {
-//                                         gitlabCommitStatus(name: 'Deloy Test') {
-//                                             container('milvus-testframework') {
-//                                                 script {
-//                                                     load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy"
-//                                                     load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy"
-//                                                 }
-//                                             }
-//                                         }
-//                                     }
-//                                 }
-//                                 stage ("Cleanup Dev") {
-//                                     steps {
-//                                         gitlabCommitStatus(name: 'Cleanup Dev') {
-//                                             container('milvus-testframework') {
-//                                                 script {
-//                                                     load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy"
-//                                                 }
-//                                             }
-//                                         }
-//                                     }
-//                                 }
-//                             }
-//                             post {
-//                                 always {
-//                                     container('milvus-testframework') {
-//                                         script {
-//                                             load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy"
-//                                         }
-//                                     }
-//                                 }
-//                                 success {
-//                                     script {
-//                                         echo "Milvus Deploy to Dev Cluster CI/CD success !"
-//                                     }
-//                                 }
-//                                 aborted {
-//                                     script {
-//                                         echo "Milvus Deploy to Dev Cluster CI/CD aborted !"
-//                                     }
-//                                 }
-//                                 failure {
-//                                     script {
-//                                         echo "Milvus Deploy to Dev Cluster CI/CD failure !"
-//                                     }
-//                                 }
-//                             }
-//                         }
-                    }
-                }
-
-                stage("Deploy to Staging") {
-                    parallel {
-                        stage("Single Node") {
-                            agent {
-                                kubernetes {
-                                    label 'dev-test'
-                                    defaultContainer 'jnlp'
-                                    yaml """
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: milvus
-    componet: test
-spec:
-  containers:
-  - name: milvus-testframework
-    image: registry.zilliz.com/milvus/milvus-test:v0.2
-    command:
-    - cat
-    tty: true
-    volumeMounts:
-    - name: kubeconf
-      mountPath: /root/.kube/
-      readOnly: true
-  volumes:
-  - name: kubeconf
-    secret:
-      secretName: aks-gpu-cluster-config
-"""
-                                }
-                            }
-
-                            stages {
-                                stage("Deploy to Staging") {
-                                    steps {
-                                        gitlabCommitStatus(name: 'Deloy to Staging') {
-                                            container('milvus-testframework') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/deploy2staging.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                                stage("Staging Test") {
-                                    steps {
-                                        gitlabCommitStatus(name: 'Staging Test') {
-                                            container('milvus-testframework') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/staging_test.groovy"
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/upload_staging_test_out.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                                stage ("Cleanup Staging") {
-                                    steps {
-                                        gitlabCommitStatus(name: 'Cleanup Staging') {
-                                            container('milvus-testframework') {
-                                                script {
-                                                    load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                            }
-                            post {
-                                always {
-                                    container('milvus-testframework') {
-                                        script {
-                                            load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
-                                        }
-                                    }
-                                }
-                                success {
-                                    script {
-                                        echo "Milvus Deploy to Staging Single Node CI/CD success !"
-                                    }
-                                }
-                                aborted {
-                                    script {
-                                        echo "Milvus Deploy to Staging Single Node CI/CD aborted !"
-                                    }
-                                }
-                                failure {
-                                    script {
-                                        echo "Milvus Deploy to Staging Single Node CI/CD failure !"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    post {
-        always {
-            script {
-                if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
-                    // Send an email only if the build status has changed from green/unstable to red
-                    emailext subject: '$DEFAULT_SUBJECT',
-                    body: '$DEFAULT_CONTENT',
-                    recipientProviders: [
-                        [$class: 'DevelopersRecipientProvider'],
-                        [$class: 'RequesterRecipientProvider']
-                    ],
-                    replyTo: '$DEFAULT_REPLYTO',
-                    to: '$DEFAULT_RECIPIENTS'
-                }
-            }
-        }
-
-        success {
-            script {
-                updateGitlabCommitStatus name: 'CI/CD', state: 'success'
-                echo "Milvus CI/CD success !"
-            }
-        }
-
-        aborted {
-            script {
-                updateGitlabCommitStatus name: 'CI/CD', state: 'canceled'
-                echo "Milvus CI/CD aborted !"
-            }
-        }
-
-        failure {
-            script {
-                updateGitlabCommitStatus name: 'CI/CD', state: 'failed'
-                echo "Milvus CI/CD failure !"
-            }
-        }
-    }
-}
-

ci/pod_containers/milvus-engine-build.yaml

-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: milvus
-    componet: build-env
-spec:
-  containers:
-  - name: milvus-build-env
-    image: registry.zilliz.com/milvus/milvus-build-env:v0.9
-    command:
-    - cat
-    tty: true

ci/pod_containers/publish-docker.yaml

-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: publish
-    componet: docker
-spec:
-  containers:
-  - name: publish-docker
-    image: registry.zilliz.com/library/zilliz_docker:v1.0.0
-    securityContext:
-      privileged: true
-    command:
-    - cat
-    tty: true
-    volumeMounts:
-    - name: docker-sock
-      mountPath: /var/run/docker.sock
-  volumes:
-  - name: docker-sock
-    hostPath:
-      path: /var/run/docker.sock

core/cmake/BuildUtils.cmake

@@ -74,7 +74,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path)
             ${CMAKE_COMMAND} -E echo
             "Extracting ${package_file} to ${install_path}"
         COMMAND
-            ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path}
+            ${CMAKE_COMMAND} -E tar xzf ${package_file} ${install_path}
             WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     )
 

core/src/scheduler/SchedInst.cpp

@@ -54,36 +54,40 @@ load_simple_config() {
 
     // get resources
 #ifdef MILVUS_GPU_VERSION
+    bool enable_gpu = false;
     server::Config& config = server::Config::GetInstance();
-    std::vector<int64_t> gpu_ids;
-    config.GetGpuResourceConfigSearchResources(gpu_ids);
-    std::vector<int64_t> build_gpu_ids;
-    config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids);
-    auto pcie = Connection("pcie", 12000);
-
-    std::vector<int64_t> not_find_build_ids;
-    for (auto& build_id : build_gpu_ids) {
-        bool find_gpu_id = false;
-        for (auto& gpu_id : gpu_ids) {
-            if (gpu_id == build_id) {
-                find_gpu_id = true;
-                break;
+    config.GetGpuResourceConfigEnable(enable_gpu);
+    if (enable_gpu) {
+        std::vector<int64_t> gpu_ids;
+        config.GetGpuResourceConfigSearchResources(gpu_ids);
+        std::vector<int64_t> build_gpu_ids;
+        config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids);
+        auto pcie = Connection("pcie", 12000);
+
+        std::vector<int64_t> not_find_build_ids;
+        for (auto& build_id : build_gpu_ids) {
+            bool find_gpu_id = false;
+            for (auto& gpu_id : gpu_ids) {
+                if (gpu_id == build_id) {
+                    find_gpu_id = true;
+                    break;
+                }
+            }
+            if (not find_gpu_id) {
+                not_find_build_ids.emplace_back(build_id);
             }
         }
-        if (not find_gpu_id) {
-            not_find_build_ids.emplace_back(build_id);
-        }
-    }
 
-    for (auto& gpu_id : gpu_ids) {
-        ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true));
-        ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie);
-    }
+        for (auto& gpu_id : gpu_ids) {
+            ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true));
+            ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie);
+        }
 
-    for (auto& not_find_id : not_find_build_ids) {
-        ResMgrInst::GetInstance()->Add(
-            ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id, true, true));
-        ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie);
+        for (auto& not_find_id : not_find_build_ids) {
+            ResMgrInst::GetInstance()->Add(
+                ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id, true, true));
+            ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie);
+        }
     }
 #endif
 }

core/src/scheduler/SchedInst.h

@@ -102,11 +102,16 @@ class OptimizerInst {
             if (instance == nullptr) {
                 std::vector<PassPtr> pass_list;
 #ifdef MILVUS_GPU_VERSION
-                pass_list.push_back(std::make_shared<BuildIndexPass>());
-                pass_list.push_back(std::make_shared<FaissFlatPass>());
-                pass_list.push_back(std::make_shared<FaissIVFFlatPass>());
-                pass_list.push_back(std::make_shared<FaissIVFSQ8Pass>());
-                pass_list.push_back(std::make_shared<FaissIVFSQ8HPass>());
+                bool enable_gpu = false;
+                server::Config& config = server::Config::GetInstance();
+                config.GetGpuResourceConfigEnable(enable_gpu);
+                if (enable_gpu) {
+                    pass_list.push_back(std::make_shared<BuildIndexPass>());
+                    pass_list.push_back(std::make_shared<FaissFlatPass>());
+                    pass_list.push_back(std::make_shared<FaissIVFFlatPass>());
+                    pass_list.push_back(std::make_shared<FaissIVFSQ8Pass>());
+                    pass_list.push_back(std::make_shared<FaissIVFSQ8HPass>());
+                }
 #endif
                 pass_list.push_back(std::make_shared<FallbackPass>());
                 instance = std::make_shared<Optimizer>(pass_list);

core/src/server/Config.cpp

@@ -189,35 +189,37 @@ Config::ValidateConfig() {
     }
 
     /* gpu resource config */
-#ifdef MILVUS_GPU_VERSION
     bool gpu_resource_enable;
     s = GetGpuResourceConfigEnable(gpu_resource_enable);
     if (!s.ok()) {
         return s;
     }
 
-    int64_t resource_cache_capacity;
-    s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity);
-    if (!s.ok()) {
-        return s;
-    }
+#ifdef MILVUS_GPU_VERSION
+    if (gpu_resource_enable) {
+        int64_t resource_cache_capacity;
+        s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity);
+        if (!s.ok()) {
+            return s;
+        }
 
-    float resource_cache_threshold;
-    s = GetGpuResourceConfigCacheThreshold(resource_cache_threshold);
-    if (!s.ok()) {
-        return s;
-    }
+        float resource_cache_threshold;
+        s = GetGpuResourceConfigCacheThreshold(resource_cache_threshold);
+        if (!s.ok()) {
+            return s;
+        }
 
-    std::vector<int64_t> search_resources;
-    s = GetGpuResourceConfigSearchResources(search_resources);
-    if (!s.ok()) {
-        return s;
-    }
+        std::vector<int64_t> search_resources;
+        s = GetGpuResourceConfigSearchResources(search_resources);
+        if (!s.ok()) {
+            return s;
+        }
 
-    std::vector<int64_t> index_build_resources;
-    s = GetGpuResourceConfigBuildIndexResources(index_build_resources);
-    if (!s.ok()) {
-        return s;
+        std::vector<int64_t> index_build_resources;
+        s = GetGpuResourceConfigBuildIndexResources(index_build_resources);
+        if (!s.ok()) {
+            return s;
+        }
     }
 #endif
 

core/src/server/grpc_impl/request/GrpcBaseRequest.cpp

@@ -90,8 +90,8 @@ GrpcBaseRequest::SetStatus(ErrorCode error_code, const std::string& error_msg) {
 std::string
 GrpcBaseRequest::TableNotExistMsg(const std::string& table_name) {
     return "Table " + table_name +
-           " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name "
-           "exists.";
+           " does not exist. Use milvus.has_table to verify whether the table exists. "
+           "You also can check whether the table name exists.";
 }
 
 Status

core/src/wrapper/KnowhereResource.cpp

@@ -37,6 +37,16 @@ constexpr int64_t M_BYTE = 1024 * 1024;
 Status
 KnowhereResource::Initialize() {
 #ifdef MILVUS_GPU_VERSION
+    Status s;
+    bool enable_gpu = false;
+    server::Config& config = server::Config::GetInstance();
+    s = config.GetGpuResourceConfigEnable(enable_gpu);
+    if (!s.ok())
+        return s;
+
+    if (not enable_gpu)
+        return Status::OK();
+
     struct GpuResourceSetting {
         int64_t pinned_memory = 300 * M_BYTE;
         int64_t temp_memory = 300 * M_BYTE;
@@ -44,10 +54,8 @@ KnowhereResource::Initialize() {
     };
     using GpuResourcesArray = std::map<int64_t, GpuResourceSetting>;
     GpuResourcesArray gpu_resources;
-    Status s;
 
     // get build index gpu resource
-    server::Config& config = server::Config::GetInstance();
     std::vector<int64_t> build_index_gpus;
     s = config.GetGpuResourceConfigBuildIndexResources(build_index_gpus);
     if (!s.ok())

tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy

+timeout(time: 7200, unit: 'MINUTES') {
+    try {
+        dir ("milvu_ann_acc") {
+            print "Git clone url: ${TEST_URL}:${TEST_BRANCH}"
+            checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]])
+            print "Install requirements"
+            sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
+            // sleep(120000)
+            sh "python3 main.py --suite=${params.SUITE} --host=acc-test-${env.JOB_NAME}-${env.BUILD_NUMBER}-engine.milvus.svc.cluster.local --port=19530"
+        }
+    } catch (exc) {
+        echo 'Milvus Ann Accuracy Test Failed !'
+        throw exc
+    }
+}
+

tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy

+timeout(time: 30, unit: 'MINUTES') {
+    try {
+        dir ("milvus") {
+            sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+            sh 'helm repo update'
+            checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]])
+            dir ("milvus") {
+                sh "helm install --wait --timeout 300 --set engine.image.tag=${IMAGE_TAG} --set expose.type=clusterIP --name acc-test-${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/sqlite_${params.IMAGE_TYPE}_values.yaml -f ci/filebeat/values.yaml --namespace milvus --version ${HELM_BRANCH} ."
+            }
+        }
+        // dir ("milvus") {
+        //     checkout([$class: 'GitSCM', branches: [[name: "${env.SERVER_BRANCH}"]], userRemoteConfigs: [[url: "${env.SERVER_URL}", name: 'origin', refspec: "+refs/heads/${env.SERVER_BRANCH}:refs/remotes/origin/${env.SERVER_BRANCH}"]]])
+        //     dir ("milvus") {
+        //         load "ci/jenkins/step/deploySingle2Dev.groovy"
+        //     }
+        // }
+    } catch (exc) {
+        echo 'Deploy Milvus Server Failed !'
+        throw exc
+    }
+}
+

tests/milvus_ann_acc/ci/main_jenkinsfile

+pipeline {
+    agent none
+
+    options {
+        timestamps()
+    }
+
+    parameters{
+        choice choices: ['cpu', 'gpu'], description: 'cpu or gpu version', name: 'IMAGE_TYPE'
+        string defaultValue: '0.6.0', description: 'server image version', name: 'IMAGE_VERSION', trim: true
+        string defaultValue: 'suite.yaml', description: 'test suite config yaml', name: 'SUITE', trim: true
+        string defaultValue: '09509e53-9125-4f5d-9ce8-42855987ad67', description: 'git credentials', name: 'GIT_USER', trim: true
+    }
+
+    environment {
+        IMAGE_TAG = "${params.IMAGE_VERSION}-${params.IMAGE_TYPE}-ubuntu18.04-release"
+        HELM_URL = "https://github.com/milvus-io/milvus-helm.git"
+        HELM_BRANCH = "0.6.0"
+        TEST_URL = "git@192.168.1.105:Test/milvus_ann_acc.git"
+        TEST_BRANCH = "0.6.0"
+    }
+
+    stages {
+        stage("Setup env") {
+            agent {
+                kubernetes {
+                    label 'dev-test'
+                    defaultContainer 'jnlp'
+                    yaml """
+                        apiVersion: v1
+                        kind: Pod
+                        metadata:
+                          labels:
+                            app: milvus
+                            componet: test
+                        spec:
+                          containers:
+                          - name: milvus-testframework
+                            image: registry.zilliz.com/milvus/milvus-test:v0.2
+                            command:
+                            - cat
+                            tty: true
+                            volumeMounts:
+                            - name: kubeconf
+                              mountPath: /root/.kube/
+                              readOnly: true
+                            - name: hdf5-path
+                              mountPath: /test
+                              readOnly: true
+                          volumes:
+                          - name: kubeconf
+                            secret:
+                              secretName: test-cluster-config
+                          - name: hdf5-path
+                            flexVolume:
+                              driver: "fstab/cifs"
+                              fsType: "cifs"
+                              secretRef:
+                                name: "cifs-test-secret"
+                              options:
+                                networkPath: "//192.168.1.126/test"
+                                mountOptions: "vers=1.0"
+                        """
+                }
+            }
+
+            stages {
+                stage("Deploy Default Server") {
+                    steps {
+                        gitlabCommitStatus(name: 'Accuracy Test') {
+                            container('milvus-testframework') {
+                                script {
+                                    print "In Deploy Default Server Stage"
+                                    load "${env.WORKSPACE}/ci/jenkinsfile/deploy_default_server.groovy"
+                                }
+                            }
+                        }
+                    }
+                }
+                stage("Acc Test") {
+                    steps {
+                        gitlabCommitStatus(name: 'Accuracy Test') {
+                            container('milvus-testframework') {
+                                script {
+                                    print "In Acc test stage"
+                                    load "${env.WORKSPACE}/ci/jenkinsfile/acc_test.groovy"
+                                }
+                            }
+                        }
+                    }
+                }
+                stage ("Cleanup Env") {
+                    steps {
+                        gitlabCommitStatus(name: 'Cleanup Env') {
+                            container('milvus-testframework') {
+                                script {
+                                    load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            post {
+                always {
+                    container('milvus-testframework') {
+                        script {
+                            load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy"
+                        }
+                    }
+                }
+                success {
+                    script {
+                        echo "Milvus ann-accuracy test success !"
+                    }
+                }
+                aborted {
+                    script {
+                        echo "Milvus ann-accuracy test aborted !"
+                    }
+                }
+                failure {
+                    script {
+                        echo "Milvus ann-accuracy test failed !"
+                    }
+                }
+            }
+        }
+    }
+}

ci/pod_containers/milvus-testframework.yaml → tests/milvus_ann_acc/ci/pod_containers/milvus-testframework.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   containers:
   - name: milvus-testframework
-    image: registry.zilliz.com/milvus/milvus-test:v0.1
+    image: registry.zilliz.com/milvus/milvus-test:v0.2
     command:
     - cat
     tty: true

tests/milvus_ann_acc/client.py

@@ -8,7 +8,7 @@ import numpy
 import sklearn.preprocessing
 from milvus import Milvus, IndexType, MetricType
 
-logger = logging.getLogger("milvus_ann_acc.client")
+logger = logging.getLogger("milvus_acc.client")
 
 SERVER_HOST_DEFAULT = "127.0.0.1"
 SERVER_PORT_DEFAULT = 19530
@@ -28,17 +28,17 @@ def time_wrapper(func):
 
 
 class MilvusClient(object):
-    def __init__(self, table_name=None, ip=None, port=None):
+    def __init__(self, table_name=None, host=None, port=None):
         self._milvus = Milvus()
         self._table_name = table_name
         try:
-            if not ip:
+            if not host:
                 self._milvus.connect(
                     host = SERVER_HOST_DEFAULT,
                     port = SERVER_PORT_DEFAULT)
             else:
                 self._milvus.connect(
-                    host = ip,
+                    host = host,
                     port = port)
         except Exception as e:
             raise e
@@ -113,7 +113,6 @@ class MilvusClient(object):
         X = X.astype(numpy.float32)
         status, results = self._milvus.search_vectors(self._table_name, top_k, nprobe, X.tolist())
         self.check_status(status)
-        # logger.info(results[0])
         ids = []
         for result in results:
             tmp_ids = []
@@ -125,24 +124,20 @@ class MilvusClient(object):
     def count(self):
         return self._milvus.get_table_row_count(self._table_name)[1]
 
-    def delete(self, timeout=60):
-        logger.info("Start delete table: %s" % self._table_name)
-        self._milvus.delete_table(self._table_name)
-        i = 0
-        while i < timeout:
-            if self.count():
-                time.sleep(1)
-                i = i + 1
-            else:
-                break
-        if i >= timeout:
-            logger.error("Delete table timeout")
+    def delete(self, table_name):
+        logger.info("Start delete table: %s" % table_name)
+        return self._milvus.delete_table(table_name)
 
     def describe(self):
         return self._milvus.describe_table(self._table_name)
 
-    def exists_table(self):
-        return self._milvus.has_table(self._table_name)
+    def exists_table(self, table_name):
+        return self._milvus.has_table(table_name)
+
+    def get_server_version(self):
+        status, res = self._milvus.server_version()
+        self.check_status(status)
+        return res
 
     @time_wrapper
     def preload_table(self):

tests/milvus_ann_acc/main.py

-
+import os
+import sys
 import argparse
+from yaml import load, dump
+import logging
+from logging import handlers
+from client import MilvusClient
+import runner
+
+LOG_FOLDER = "logs"
+logger = logging.getLogger("milvus_acc")
+formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s')
+if not os.path.exists(LOG_FOLDER):
+    os.system('mkdir -p %s' % LOG_FOLDER)
+fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'acc'), "D", 1, 10)
+fileTimeHandler.suffix = "%Y%m%d.log"
+fileTimeHandler.setFormatter(formatter)
+logging.basicConfig(level=logging.DEBUG)
+fileTimeHandler.setFormatter(formatter)
+logger.addHandler(fileTimeHandler)
+
 
 def main():
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument(
-        '--dataset',
-        metavar='NAME',
-        help='the dataset to load training points from',
-        default='glove-100-angular',
-        choices=DATASETS.keys())
+        "--host",
+        default="127.0.0.1",
+        help="server host")
     parser.add_argument(
-        "-k", "--count",
-        default=10,
-        type=positive_int,
-        help="the number of near neighbours to search for")
+        "--port",
+        default=19530,
+        help="server port")   
     parser.add_argument(
-        '--definitions',
+        '--suite',
         metavar='FILE',
-        help='load algorithm definitions from FILE',
-        default='algos.yaml')
-    parser.add_argument(
-        '--image-tag',
-        default=None,
-        help='pull image first')
\ No newline at end of file
+        help='load config definitions from suite_czr'
+             '.yaml',
+        default='suite_czr.yaml')
+    args = parser.parse_args()
+    if args.suite:
+        with open(args.suite, "r") as f:
+            suite = load(f)
+            hdf5_path = suite["hdf5_path"]
+            dataset_configs = suite["datasets"]
+            if not hdf5_path or not dataset_configs:
+                logger.warning("No datasets given")
+                sys.exit()
+            f.close()
+    for dataset_config in dataset_configs:
+        logger.debug(dataset_config)
+        milvus_instance = MilvusClient(host=args.host, port=args.port)
+        runner.run(milvus_instance, dataset_config, hdf5_path)
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

tests/milvus_ann_acc/requirements.txt

@@ -2,3 +2,8 @@ numpy==1.16.3
 pymilvus>=0.2.0
 scikit-learn==0.19.1
 h5py==2.7.1
+influxdb==5.2.2
+pyyaml==3.12
+tableprint==0.8.0
+ansicolors==1.1.8
+scipy==1.3.1
\ No newline at end of file

tests/milvus_ann_acc/runner.py

+import os
+import pdb
+import time
+import random
+import sys
+import logging
+import h5py
+import numpy
+from influxdb import InfluxDBClient
+
+INSERT_INTERVAL = 100000
+# s
+DELETE_INTERVAL_TIME = 5
+INFLUXDB_HOST = "192.168.1.194"
+INFLUXDB_PORT = 8086
+INFLUXDB_USER = "admin"
+INFLUXDB_PASSWD = "admin"
+INFLUXDB_NAME = "test_result"
+influxdb_client = InfluxDBClient(host=INFLUXDB_HOST, port=INFLUXDB_PORT, username=INFLUXDB_USER, password=INFLUXDB_PASSWD, database=INFLUXDB_NAME)
+
+logger = logging.getLogger("milvus_acc.runner")
+
+
+def parse_dataset_name(dataset_name):
+    data_type = dataset_name.split("-")[0]
+    dimension = int(dataset_name.split("-")[1])
+    metric = dataset_name.split("-")[-1]
+    # metric = dataset.attrs['distance']
+    # dimension = len(dataset["train"][0])
+    if metric == "euclidean":
+        metric_type = "l2"
+    elif metric  == "angular":
+        metric_type = "ip"
+    return ("ann"+data_type, dimension, metric_type)
+
+
+def get_dataset(hdf5_path, dataset_name):
+    file_path = os.path.join(hdf5_path, '%s.hdf5' % dataset_name)
+    if not os.path.exists(file_path):
+        raise Exception("%s not existed" % file_path)
+    dataset = h5py.File(file_path)
+    return dataset
+
+
+def get_table_name(hdf5_path, dataset_name, index_file_size):
+    data_type, dimension, metric_type = parse_dataset_name(dataset_name)
+    dataset = get_dataset(hdf5_path, dataset_name)
+    table_size = len(dataset["train"])
+    table_size = str(table_size // 1000000)+"m"
+    table_name = data_type+'_'+table_size+'_'+str(index_file_size)+'_'+str(dimension)+'_'+metric_type
+    return table_name
+
+
+def recall_calc(result_ids, true_ids, top_k, recall_k):
+    sum_intersect_num = 0
+    recall = 0.0
+    for index, result_item in enumerate(result_ids):
+        if len(set(true_ids[index][:top_k])) != len(set(result_item)):
+            logger.warning("Error happened: query result length is wrong")
+            continue
+        tmp = set(true_ids[index][:recall_k]).intersection(set(result_item))
+        sum_intersect_num = sum_intersect_num + len(tmp)
+    recall = round(sum_intersect_num / (len(result_ids) * recall_k), 4)
+    return recall
+
+
+def run(milvus, config, hdf5_path, force=True):
+    server_version = milvus.get_server_version()
+    logger.info(server_version)
+    
+    for dataset_name, config_value in config.items():
+        dataset = get_dataset(hdf5_path, dataset_name)
+        index_file_sizes = config_value["index_file_sizes"]
+        index_types = config_value["index_types"]
+        nlists = config_value["nlists"]
+        search_param = config_value["search_param"]
+        top_ks = search_param["top_ks"]
+        nprobes = search_param["nprobes"]
+        nqs = search_param["nqs"]
+
+        for index_file_size in index_file_sizes:
+            table_name = get_table_name(hdf5_path, dataset_name, index_file_size)  
+            if milvus.exists_table(table_name):
+                if force is True:
+                    logger.info("Re-create table: %s" % table_name)
+                    milvus.delete(table_name)
+                    time.sleep(DELETE_INTERVAL_TIME)
+                else:
+                    logger.warning("Table name: %s existed" % table_name)
+                    continue
+            data_type, dimension, metric_type = parse_dataset_name(dataset_name)
+            milvus.create_table(table_name, dimension, index_file_size, metric_type)
+            logger.info(milvus.describe())
+            insert_vectors = numpy.array(dataset["train"])
+            # milvus.insert(insert_vectors)
+
+            loops = len(insert_vectors) // INSERT_INTERVAL + 1
+            for i in range(loops):
+                start = i*INSERT_INTERVAL
+                end = min((i+1)*INSERT_INTERVAL, len(insert_vectors))
+                tmp_vectors = insert_vectors[start:end]
+                if start < end:
+                    milvus.insert(tmp_vectors, ids=[i for i in range(start, end)])
+            time.sleep(20)
+            row_count = milvus.count()
+            logger.info("Table: %s, row count: %s" % (table_name, row_count))
+            if milvus.count() != len(insert_vectors):
+                logger.error("Table row count is not equal to insert vectors")
+                return
+            for index_type in index_types:
+                for nlist in nlists:
+                    milvus.create_index(index_type, nlist)
+                    logger.info(milvus.describe_index())
+                    logger.info("Start preload table: %s, index_type: %s, nlist: %s" % (table_name, index_type, nlist))
+                    milvus.preload_table()
+                    true_ids = numpy.array(dataset["neighbors"])
+                    for nprobe in nprobes:
+                        for nq in nqs:
+                            query_vectors = numpy.array(dataset["test"][:nq])
+                            for top_k in top_ks:
+                                rec1 = 0.0
+                                rec10 = 0.0
+                                rec100 = 0.0
+                                result_ids = milvus.query(query_vectors, top_k, nprobe)
+                                logger.info("Query result: %s" % len(result_ids))
+                                rec1 = recall_calc(result_ids, true_ids, top_k, 1)
+                                if top_k == 10:
+                                    rec10 = recall_calc(result_ids, true_ids, top_k, 10)
+                                if top_k == 100:
+                                    rec10 = recall_calc(result_ids, true_ids, top_k, 10)
+                                    rec100 = recall_calc(result_ids, true_ids, top_k, 100)
+                                avg_radio = recall_calc(result_ids, true_ids, top_k, top_k)
+                                logger.debug("Recall_1: %s" % rec1)
+                                logger.debug("Recall_10: %s" % rec10)
+                                logger.debug("Recall_100: %s" % rec100)
+                                logger.debug("Accuracy: %s" % avg_radio)
+                                acc_record = [{
+                                    "measurement": "accuracy",
+                                    "tags": {
+                                        "server_version": server_version,
+                                        "dataset": dataset_name,
+                                        "index_file_size": index_file_size,
+                                        "index_type": index_type,
+                                        "nlist": nlist,
+                                        "search_nprobe": nprobe,
+                                        "top_k": top_k,
+                                        "nq": len(query_vectors)
+                                    },
+                                    # "time": time.ctime(),
+                                    "time": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
+                                    "fields": {
+                                        "recall1": rec1,
+                                        "recall10": rec10,
+                                        "recall100": rec100,
+                                        "avg_radio": avg_radio
+                                    }
+                                }]
+                                logger.info(acc_record)
+                                try:
+                                    res = influxdb_client.write_points(acc_record)
+                                except Exception as e:
+                                    logger.error("Insert infuxdb failed: %s" % str(e))

tests/milvus_ann_acc/suite.yaml

+datasets:
+  - sift-128-euclidean: 
+      index_file_sizes: [50, 1024]
+      index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h']
+      # index_types: ['ivf_sq8']
+      nlists: [16384]
+      search_param:
+        nprobes: [1, 32, 128, 256]
+        top_ks: [10]
+        nqs: [10000]
+  - glove-25-angular:
+      index_file_sizes: [50, 1024]
+      index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h']
+      # index_types: ['ivf_sq8']
+      nlists: [16384]
+      search_param:
+        nprobes: [1, 32, 128, 256]
+        top_ks: [10]
+        nqs: [10000]
+  - glove-200-angular:
+      index_file_sizes: [50, 1024]
+      index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h']
+      # index_types: ['ivf_sq8']
+      nlists: [16384]
+      search_param:
+        nprobes: [1, 32, 128, 256]
+        top_ks: [10]
+        nqs: [10000]
+hdf5_path: /test/milvus/ann_hdf5/
\ No newline at end of file

tests/milvus_ann_acc/suite.yaml.bak

+datasets:
+  - glove-200-angular:
+      index_file_sizes: [1024]
+      index_types: ['ivf_sq8']
+      # index_types: ['ivf_sq8']
+      nlists: [16384]
+      search_param:
+        nprobes: [256, 400, 256]
+        top_ks: [100]
+        nqs: [10000]
+hdf5_path: /test/milvus/ann_hdf5/

tests/milvus_ann_acc/suite_czr.yaml

+datasets:
+  - sift-128-euclidean:
+      index_file_sizes: [1024]
+      index_types: ['ivf_sq8', 'ivf_sq8h']
+      # index_types: ['ivf_sq8']
+      nlists: [16384]
+      search_param:
+        nprobes: [16, 128, 1024]
+        top_ks: [1, 10, 100]
+        nqs: [10, 100, 1000]
+  - glove-200-angular:
+      index_file_sizes: [1024]
+      index_types: ['ivf_sq8', 'ivf_sq8h']
+      # index_types: ['ivf_sq8']
+      nlists: [16384]
+      search_param:
+        nprobes: [16, 128, 1024]
+        top_ks: [1, 10, 100]
+        nqs: [10, 100, 1000]
+hdf5_path: /test/milvus/ann_hdf5/
\ No newline at end of file

tests/milvus_ann_acc/suite_debug.yaml

+datasets:
+  - sift-128-euclidean: 
+      index_file_sizes: [1024]
+      index_types: ['ivf_flat']
+      nlists: [16384]
+      search_param:
+        nprobes: [1, 256]
+        top_ks: [10]
+        nqs: [10000]
+hdf5_path: /test/milvus/ann_hdf5/

tests/milvus_ann_acc/test.py

-import os
-import pdb
 import time
-import random
-import sys
-import h5py
-import numpy
-import logging
-from logging import handlers
-
-from client import MilvusClient
-
-LOG_FOLDER = "logs"
-logger = logging.getLogger("milvus_ann_acc")
-
-formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s')
-if not os.path.exists(LOG_FOLDER):
-    os.system('mkdir -p %s' % LOG_FOLDER)
-fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'acc'), "D", 1, 10)
-fileTimeHandler.suffix = "%Y%m%d.log"
-fileTimeHandler.setFormatter(formatter)
-logging.basicConfig(level=logging.DEBUG)
-fileTimeHandler.setFormatter(formatter)
-logger.addHandler(fileTimeHandler)
-
-
-def get_dataset_fn(dataset_name):
-    file_path = "/test/milvus/ann_hdf5/"
-    if not os.path.exists(file_path):
-        raise Exception("%s not exists" % file_path)
-    return os.path.join(file_path, '%s.hdf5' % dataset_name)
-
-
-def get_dataset(dataset_name):
-    hdf5_fn = get_dataset_fn(dataset_name)
-    hdf5_f = h5py.File(hdf5_fn)
-    return hdf5_f
-
-
-def parse_dataset_name(dataset_name):
-    data_type = dataset_name.split("-")[0]
-    dimension = int(dataset_name.split("-")[1])
-    metric = dataset_name.split("-")[-1]
-    # metric = dataset.attrs['distance']
-    # dimension = len(dataset["train"][0])
-    if metric == "euclidean":
-        metric_type = "l2"
-    elif metric  == "angular":
-        metric_type = "ip"
-    return ("ann"+data_type, dimension, metric_type)
-
-
-def get_table_name(dataset_name, index_file_size):
-    data_type, dimension, metric_type = parse_dataset_name(dataset_name)
-    dataset = get_dataset(dataset_name)
-    table_size = len(dataset["train"])
-    table_size = str(table_size // 1000000)+"m"
-    table_name = data_type+'_'+table_size+'_'+str(index_file_size)+'_'+str(dimension)+'_'+metric_type
-    return table_name
-
-
-def main(dataset_name, index_file_size, nlist=16384, force=False):
-    top_k = 10
-    nprobes = [32, 128]
-
-    dataset = get_dataset(dataset_name)
-    table_name = get_table_name(dataset_name, index_file_size)
-    m = MilvusClient(table_name)
-    if m.exists_table():
-        if force is True:
-            logger.info("Re-create table: %s" % table_name)
-            m.delete()
-            time.sleep(10)
-        else:
-            logger.info("Table name: %s existed" % table_name)
-            return
-    data_type, dimension, metric_type = parse_dataset_name(dataset_name)
-    m.create_table(table_name, dimension, index_file_size, metric_type)
-    print(m.describe())
-    vectors = numpy.array(dataset["train"])
-    query_vectors = numpy.array(dataset["test"])
-    # m.insert(vectors)
-
-    interval = 100000
-    loops = len(vectors) // interval + 1
-
-    for i in range(loops):
-        start = i*interval
-        end = min((i+1)*interval, len(vectors))
-        tmp_vectors = vectors[start:end]
-        if start < end:
-            m.insert(tmp_vectors, ids=[i for i in range(start, end)])
-    
-    time.sleep(60)
-    print(m.count())
-
-    for index_type in ["ivf_flat", "ivf_sq8", "ivf_sq8h"]:
-        m.create_index(index_type, nlist)
-        print(m.describe_index())
-        if m.count() != len(vectors):
-            return
-        m.preload_table()
-        true_ids = numpy.array(dataset["neighbors"])
-        for nprobe in nprobes:
-            print("nprobe: %s" % nprobe)
-            sum_radio = 0.0; avg_radio = 0.0
-            result_ids = m.query(query_vectors, top_k, nprobe)
-            # print(result_ids[:10])
-            for index, result_item in enumerate(result_ids):
-                if len(set(true_ids[index][:top_k])) != len(set(result_item)):
-                    logger.info("Error happened")
-                    # logger.info(query_vectors[index])
-                    # logger.info(true_ids[index][:top_k], result_item)
-                tmp = set(true_ids[index][:top_k]).intersection(set(result_item))
-                sum_radio = sum_radio + (len(tmp) / top_k)
-            avg_radio = round(sum_radio / len(result_ids), 4) 
-            logger.info(avg_radio)
-        m.drop_index()
-
-
-if __name__ == "__main__":
-    print("glove-25-angular")
-    # main("sift-128-euclidean", 1024, force=True)
-    for index_file_size in [50, 1024]:
-        print("Index file size: %d" % index_file_size)
-        main("glove-25-angular", index_file_size, force=True)
-
-    print("sift-128-euclidean")
-    for index_file_size in [50, 1024]:
-        print("Index file size: %d" % index_file_size)
-        main("sift-128-euclidean", index_file_size, force=True)
-    # m = MilvusClient()
\ No newline at end of file
+from influxdb import InfluxDBClient
+
+INFLUXDB_HOST = "192.168.1.194"
+INFLUXDB_PORT = 8086
+INFLUXDB_USER = "admin"
+INFLUXDB_PASSWD = "admin"
+INFLUXDB_NAME = "test_result"
+
+client = InfluxDBClient(host=INFLUXDB_HOST, port=INFLUXDB_PORT, username=INFLUXDB_USER, password=INFLUXDB_PASSWD, database=INFLUXDB_NAME)
+
+print(client.get_list_database())
+acc_record = [{
+    "measurement": "accuracy",
+    "tags": {
+        "server_version": "0.4.3",
+        "dataset": "test",
+        "index_type": "test",
+        "nlist": 12,
+        "search_nprobe": 12,
+        "top_k": 1,
+        "nq": 1
+    },
+    "time": time.ctime(),
+    "fields": {
+        "accuracy": 0.1
+    }
+}]
+try:
+	res = client.write_points(acc_record)
+	print(res)
+except Exception as e:
+    print(str(e))
\ No newline at end of file

tests/milvus_python_test/test_connect.py

@@ -149,7 +149,8 @@ class TestConnect:
             milvus.connect(uri=uri_value, timeout=1)
         assert not milvus.connected()
 
-    def test_connect_with_multiprocess(self, args):
+    # disable
+    def _test_connect_with_multiprocess(self, args):
         '''
         target: test uri connect with multiprocess
         method: set correct uri, test with multiprocessing connecting

tests/milvus_python_test/test_mix.py

@@ -25,7 +25,8 @@ index_params = {'index_type': IndexType.IVFLAT, 'nlist': 16384}
 
 class TestMixBase:
 
-    def test_search_during_createIndex(self, args):
+    # disable
+    def _test_search_during_createIndex(self, args):
         loops = 10000
         table = gen_unique_str()
         query_vecs = [vectors[0], vectors[1]]