#1656 Add timeout for whole Jenkins job (#1676)

* Increase memory limit in Jenkins CI
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Add pod tolerations to slave pods

* Refactor ci/scripts/build.sh

* Refactor ci/scripts/build.sh

* Refactor CI

* Refactor CI

* Refactor CI

* Refactor CI

* Refactor CI

* Refactor CI

* Refactor CI

* Add timeout for whole Jenkins job

* cuda yum repository temporarily disabled

ci/jenkins/Jenkinsfile

@@ -13,6 +13,7 @@ pipeline {
 
     options {
         timestamps()
+        timeout(time: 3, unit: 'HOURS')
     }
 
     parameters{
@@ -64,22 +65,6 @@ pipeline {
 
                         stages {
                             stage('Run Build') {
-                                // when {
-                                //     allOf {
-                                //         anyOf {
-                                //             expression { isEmptyChangelog() == true }
-                                //             changeset 'ci/**'
-                                //             changeset 'core/**'
-                                //             changeset 'docker/**'
-                                //             changeset 'test/**'
-                                //         }
-                                //         not {
-                                //             anyOf {
-                                //                 changeset '**/*.md'
-                                //             }
-                                //         }
-                                //     }
-                                // }
                                 agent {
                                     kubernetes {
                                         label "${OS_NAME}-${BINARY_VERSION}-build-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
@@ -140,22 +125,6 @@ pipeline {
                             }
 
                             stage('Publish docker images') {
-                                // when {
-                                //     allOf {
-                                //         anyOf {
-                                //             expression { isEmptyChangelog() == true }
-                                //             changeset 'ci/**'
-                                //             changeset 'core/**'
-                                //             changeset 'docker/**'
-                                //             changeset 'test/**'
-                                //         }
-                                //         not {
-                                //             anyOf {
-                                //                 changeset '**/*.md'
-                                //             }
-                                //         }
-                                //     }
-                                // }
                                 agent {
                                     kubernetes {
                                         label "${OS_NAME}-${BINARY_VERSION}-publish-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
@@ -178,22 +147,6 @@ pipeline {
                             }
 
                             stage('Deploy to Development') {
-                                // when {
-                                //     allOf {
-                                //         anyOf {
-                                //             expression { isEmptyChangelog() == true }
-                                //             changeset 'ci/**'
-                                //             changeset 'core/**'
-                                //             changeset 'docker/**'
-                                //             changeset 'test/**'
-                                //         }
-                                //         not {
-                                //             anyOf {
-                                //                 changeset '**/*.md'
-                                //             }
-                                //         }
-                                //     }
-                                // }
                                 environment {
                                     FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-").replaceAll("_", "-")
                                     FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-")

ci/jenkins/step/build.groovy

-timeout(time: 75, unit: 'MINUTES') {
-    dir ("ci/scripts") {
-        withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-            def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true)
+dir ("ci/scripts") {
+    withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
+        def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true)
 
-            if ("${BINARY_VERSION}" == "gpu") {
-                sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -g -u\""
-            } else {
-                sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u\""
-            }
-            sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}"
+        if ("${BINARY_VERSION}" == "gpu") {
+            sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -g -u\""
+        } else {
+            sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u\""
         }
+        sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}"
     }
 }
+

ci/jenkins/step/coverage.groovy

-timeout(time: 10, unit: 'MINUTES') {
-    dir ("ci/scripts") {
-        sh "./coverage.sh"
-        boolean isNightlyTest = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0 ? true : false
-        String formatFlag = "${BINARY_VERSION}-version-${OS_NAME}-unittest".replaceAll("\\.", "_").replaceAll("-", "_")
-        if (isNightlyTest) {
-	        withCredentials([[$class: 'StringBinding', credentialsId: "milvus-ci-codecov-token", variable: 'CODECOV_TOKEN']]) {
-	            sh "curl -s https://codecov.io/bash | bash -s - -f output_new.info -n ${BINARY_VERSION}-version-${OS_NAME}-unittest -F nightly -F ${formatFlag} || echo \"Codecov did not collect coverage reports\""
-	        }
-	    } else {
-            withCredentials([[$class: 'StringBinding', credentialsId: "milvus-ci-codecov-token", variable: 'CODECOV_TOKEN']]) {
-	            sh "curl -s https://codecov.io/bash | bash -s - -f output_new.info -n ${BINARY_VERSION}-version-${OS_NAME}-unittest -F ${formatFlag} || echo \"Codecov did not collect coverage reports\""
-	        }
-	    }
+dir ("ci/scripts") {
+    sh "./coverage.sh"
+    boolean isNightlyTest = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0 ? true : false
+    String formatFlag = "${BINARY_VERSION}-version-${OS_NAME}-unittest".replaceAll("\\.", "_").replaceAll("-", "_")
+    if (isNightlyTest) {
+        withCredentials([[$class: 'StringBinding', credentialsId: "milvus-ci-codecov-token", variable: 'CODECOV_TOKEN']]) {
+            sh "curl -s https://codecov.io/bash | bash -s - -f output_new.info -n ${BINARY_VERSION}-version-${OS_NAME}-unittest -F nightly -F ${formatFlag} || echo \"Codecov did not collect coverage reports\""
+        }
+    } else {
+        withCredentials([[$class: 'StringBinding', credentialsId: "milvus-ci-codecov-token", variable: 'CODECOV_TOKEN']]) {
+            sh "curl -s https://codecov.io/bash | bash -s - -f output_new.info -n ${BINARY_VERSION}-version-${OS_NAME}-unittest -F ${formatFlag} || echo \"Codecov did not collect coverage reports\""
+        }
     }
 }

ci/jenkins/step/internalCoverage.groovy

-timeout(time: 30, unit: 'MINUTES') {
-    dir ("ci/scripts") {
-        sh "./coverage.sh -o ${env.MILVUS_INSTALL_PREFIX} -u root -p 123456 -t \$POD_IP"
-    }
-}
-

ci/jenkins/step/package.groovy

-timeout(time: 5, unit: 'MINUTES') {
-    // dir ("ci/jenkins/scripts") {
-    //     sh "pip3 install -r requirements.txt"
-    //     sh "./yaml_processor.py merge -f ${env.MILVUS_INSTALL_PREFIX}/conf/server_config.yaml -m ../yaml/update_server_config.yaml -i && rm ${env.MILVUS_INSTALL_PREFIX}/conf/server_config.yaml.bak"
-    // }
-
-    sh "rm -rf ${env.MILVUS_INSTALL_PREFIX}/unittest"
-    sh "tar -zcvf ./${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz -C ${env.MILVUS_ROOT_PATH}/ ${env.PROJECT_NAME}"
-    withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
-        def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz")
-        if (uploadStatus != 0) {
-            error("\" ${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz \" upload to \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz \" failed!")
-        }
+sh "rm -rf ${env.MILVUS_INSTALL_PREFIX}/unittest"
+sh "tar -zcvf ./${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz -C ${env.MILVUS_ROOT_PATH}/ ${env.PROJECT_NAME}"
+withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
+    def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz")
+    if (uploadStatus != 0) {
+        error("\" ${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz \" upload to \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz \" failed!")
     }
 }

ci/jenkins/step/publishImages.groovy

-timeout(time: 15, unit: 'MINUTES') {
-    dir ("docker/deploy/${BINARY_VERSION}/${OS_NAME}") {
-        def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz"
+dir ("docker/deploy/${BINARY_VERSION}/${OS_NAME}") {
+    def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz"
 
-        withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
-            def downloadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage}")
+    withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
+        def downloadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage}")
 
-            if (downloadStatus != 0) {
-                error("\" Download \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage} \" failed!")
-            }
+        if (downloadStatus != 0) {
+            error("\" Download \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage} \" failed!")
         }
-        sh "tar zxvf ${binaryPackage}"
-        def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}"
+    }
+    sh "tar zxvf ${binaryPackage}"
+    def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}"
 
-        try {
-            deleteImages("${imageName}", true)
+    try {
+        deleteImages("${imageName}", true)
 
-            def customImage = docker.build("${imageName}")
+        def customImage = docker.build("${imageName}")
 
-            deleteImages("${params.DOKCER_REGISTRY_URL}/${imageName}", true)
+        deleteImages("${params.DOKCER_REGISTRY_URL}/${imageName}", true)
 
-            docker.withRegistry("https://${params.DOKCER_REGISTRY_URL}", "${params.DOCKER_CREDENTIALS_ID}") {
-                customImage.push()
-            }
-        } catch (exc) {
-            throw exc
-        } finally {
-            deleteImages("${imageName}", true)
-            deleteImages("${params.DOKCER_REGISTRY_URL}/${imageName}", true)
+        docker.withRegistry("https://${params.DOKCER_REGISTRY_URL}", "${params.DOCKER_CREDENTIALS_ID}") {
+            customImage.push()
         }
+    } catch (exc) {
+        throw exc
+    } finally {
+        deleteImages("${imageName}", true)
+        deleteImages("${params.DOKCER_REGISTRY_URL}/${imageName}", true)
     }
 }
 

ci/jenkins/step/singleDevNightlyTest.groovy

-timeout(time: 90, unit: 'MINUTES') {
-    dir ("tests/milvus_python_test") {
-        // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
-        sh 'python3 -m pip install -r requirements.txt'
-        sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
-    }
-    // mysql database backend test
-    load "ci/jenkins/step/cleanupSingleDev.groovy"
+dir ("tests/milvus_python_test") {
+    // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
+    sh 'python3 -m pip install -r requirements.txt'
+    sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
+}
+// mysql database backend test
+load "ci/jenkins/step/cleanupSingleDev.groovy"
 
-    if (!fileExists('milvus-helm')) {
-        dir ("milvus-helm") {
-            checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
-        }
-    }
+if (!fileExists('milvus-helm')) {
     dir ("milvus-helm") {
-        sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
-    }
-    dir ("tests/milvus_python_test") {
-        sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
+        checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
     }
 }
+dir ("milvus-helm") {
+    sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
+}
+dir ("tests/milvus_python_test") {
+    sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
+}

ci/jenkins/step/singleDevTest.groovy

-timeout(time: 60, unit: 'MINUTES') {
-    dir ("tests/milvus_python_test") {
-        // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
-        sh 'python3 -m pip install -r requirements.txt'
-        sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
-    }
-
-    // mysql database backend test
-    // load "ci/jenkins/step/cleanupSingleDev.groovy"
-
-    // if (!fileExists('milvus-helm')) {
-    //     dir ("milvus-helm") {
-    //         checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
-    //     }
-    // }
-    // dir ("milvus-helm") {
-    //     sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
-    // }
-    // dir ("tests/milvus_python_test") {
-    //     sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
-    // }
+dir ("tests/milvus_python_test") {
+    // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
+    sh 'python3 -m pip install -r requirements.txt'
+    sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
 }

ci/jenkins/step/unittest.groovy

-timeout(time: 20, unit: 'MINUTES') {
-    dir ("ci/scripts") {
-        sh "./run_unittest.sh -i ${env.MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=\$POD_IP"
-    }
-}
\ No newline at end of file
+dir ("ci/scripts") {
+    sh "./run_unittest.sh -i ${env.MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=\$POD_IP"
+}

docker/deploy/gpu/centos7/Dockerfile

@@ -13,8 +13,8 @@ FROM nvidia/cuda:10.1-devel-centos7
 
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 
-RUN yum install -y epel-release && \
-    yum install -y libgomp libgfortran4 mysql-devel openblas-devel lapack-devel && \
+RUN yum --disablerepo=cuda install -y epel-release && \
+    yum --disablerepo=cuda install -y libgomp libgfortran4 mysql-devel openblas-devel lapack-devel && \
     rm -rf /var/cache/yum/*
 
 COPY ./milvus /var/lib/milvus