MS-161 - Add CI / CD Module to Milvus Project

Former-commit-id: 251e2ffd798f653920767f1550d70b2ee729bdff

CHANGELOGS.md

@@ -15,3 +15,4 @@ Please mark all change in change log and use the ticket from JIRA.
 ### Task
 
 - MS-1 - Add CHANGELOG.md
+- MS-161 - Add CI / CD Module to Milvus Project

ci/function/file_transfer.groovy

+def FileTransfer (sourceFiles, remoteDirectory, remoteIP, protocol = "ftp", makeEmptyDirs = true) {
+    if (protocol == "ftp") {
+        ftpPublisher masterNodeName: '', paramPublish: [parameterName: ''], alwaysPublishFromMaster: false, continueOnError: false, failOnError: true, publishers: [
+            [configName: "${remoteIP}", transfers: [
+                [asciiMode: false, cleanRemote: false, excludes: '', flatten: false, makeEmptyDirs: "${makeEmptyDirs}", noDefaultExcludes: false, patternSeparator: '[, ]+', remoteDirectory: "${remoteDirectory}", remoteDirectorySDF: false, removePrefix: '', sourceFiles: "${sourceFiles}"]], usePromotionTimestamp: true, useWorkspaceInPromotion: false, verbose: true
+                ]
+            ]
+    }
+}
+return this

ci/jenkinsfile/cleanup_dev.groovy

+try {
+    sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
+
+    if (currentBuild.result == 'ABORTED') {
+        throw new hudson.AbortException("Dev Test Aborted !")
+    } else if (currentBuild.result == 'FAILURE') {
+        error("Dev Test Failure !")
+    }
+} catch (exc) {
+    updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed'
+    throw exc
+}

ci/jenkinsfile/deploy2dev.groovy

+try {
+    sh 'helm init --client-only --skip-refresh'
+    sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus'
+    sh 'helm repo update'
+    sh "helm install --set engine.image.repository=registry.zilliz.com/${PROJECT_NAME}/engine --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.0 milvus/milvus-gpu"
+} catch (exc) {
+    updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed'
+    echo 'Helm running failed!'
+    sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
+    throw exc
+}

ci/jenkinsfile/dev_test.groovy

+container('milvus-testframework') {
+    timeout(time: 10, unit: 'MINUTES') {
+        gitlabCommitStatus(name: 'Dev Test') {
+            try {
+                dir ("${PROJECT_NAME}_test") {
+                    checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git"]]])
+                    sh 'python3 -m pip install -r requirements.txt'
+                    sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local"
+                }
+            } catch (exc) {
+                updateGitlabCommitStatus name: 'Dev Test', state: 'failed'
+                currentBuild.result = 'FAILURE'
+                echo 'Milvus Test Failed !'
+            }
+        }
+    }
+}

ci/jenkinsfile/milvus_build.groovy

+container('milvus-build-env') {
+    timeout(time: 20, unit: 'MINUTES') {
+        gitlabCommitStatus(name: 'Build Engine') {
+            dir ("milvus_engine") {
+                try {
+                    checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git"]]])
+                    dir ("cpp") {
+                        sh "./build.sh -t ${params.BUILD_TYPE} -u -c"
+                    }
+                } catch (exc) {
+                    updateGitlabCommitStatus name: 'Build Engine', state: 'failed'
+                    throw exc
+                }
+            }
+        }
+    }
+}

ci/jenkinsfile/milvus_build_no_ut.groovy

+container('milvus-build-env') {
+    timeout(time: 20, unit: 'MINUTES') {
+        gitlabCommitStatus(name: 'Build Engine') {
+            dir ("milvus_engine") {
+                try {
+                    checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git"]]])
+                    dir ("cpp") {
+                        sh "./build.sh -t ${params.BUILD_TYPE}"
+                    }
+                } catch (exc) {
+                    updateGitlabCommitStatus name: 'Build Engine', state: 'failed'
+                    throw exc
+                }
+            }
+        }
+    }
+}

ci/jenkinsfile/packaged_milvus.groovy

+container('milvus-build-env') {
+    timeout(time: 5, unit: 'MINUTES') {
+        dir ("milvus_engine") {
+            dir ("cpp") {
+                gitlabCommitStatus(name: 'Packaged Engine') {
+                    if (fileExists('milvus')) {
+                        try {
+                            sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus"
+                            def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
+                            fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage')
+                            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
+                                echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\""
+                            }
+                        } catch (exc) {
+                            updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed'
+                            throw exc
+                        }
+                    } else {
+                        updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed'
+                        error("Milvus binary directory don't exists!")
+                    }
+                }
+
+                gitlabCommitStatus(name: 'Packaged Engine lcov') {
+                    if (fileExists('lcov_out')) {
+                        try {
+                            def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
+                            fileTransfer.FileTransfer("lcov_out/", "${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}", 'nas storage')
+                            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
+                                echo "Milvus lcov out Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}/lcov_out/\""
+                            }
+                        } catch (exc) {
+                            updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed'
+                            throw exc
+                        }
+                    } else {
+                        updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed'
+                        error("Milvus lcov out directory don't exists!")
+                    }
+                }
+            }
+        }
+    }
+}

ci/jenkinsfile/packaged_milvus_no_ut.groovy

+container('milvus-build-env') {
+    timeout(time: 5, unit: 'MINUTES') {
+        dir ("milvus_engine") {
+            dir ("cpp") {
+                gitlabCommitStatus(name: 'Packaged Engine') {
+                    if (fileExists('milvus')) {
+                        try {
+                            sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus"
+                            def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
+                            fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage')
+                            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
+                                echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\""
+                            }
+                        } catch (exc) {
+                            updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed'
+                            throw exc
+                        }
+                    } else {
+                        updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed'
+                        error("Milvus binary directory don't exists!")
+                    }
+                }
+            }
+        }
+    }
+}

ci/jenkinsfile/publish_docker.groovy

+container('publish-docker') {
+    timeout(time: 15, unit: 'MINUTES') {
+        gitlabCommitStatus(name: 'Publish Engine Docker') {
+            try {
+                dir ("${PROJECT_NAME}_build") {
+                    checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git"]]])
+                    dir ("docker/deploy/ubuntu16.04/free_version") {
+                        sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz"
+                        sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz"
+                        try {
+                            docker.withRegistry('https://registry.zilliz.com', 'a54e38ef-c424-4ea9-9224-b25fc20e3924') {
+                                def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}")
+                                customImage.push()
+                            }
+                            echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}"
+                        } catch (exc) {
+                            updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled'
+                            throw exc
+                        } finally {
+                            sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}"
+                        }
+                    }
+                }
+            } catch (exc) {
+                updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed'
+                echo 'Publish docker failed!'
+                throw exc
+            }
+        }
+    }
+}

ci/jenkinsfile/upload_dev_test_out.groovy

+container('milvus-testframework') {
+    timeout(time: 5, unit: 'MINUTES') {
+        dir ("${PROJECT_NAME}_test") {
+            gitlabCommitStatus(name: 'Upload Dev Test Out') {
+                if (fileExists('test_out')) {
+                    try {
+                        def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
+                        fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
+                        if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
+                            echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
+                        }
+                    } catch (hudson.AbortException ae) {
+                        updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'canceled'
+                        currentBuild.result = 'ABORTED'
+                    } catch (exc) {
+                        updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed'
+                        currentBuild.result = 'FAILURE'
+                    }
+                } else {
+                    updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed'
+                    echo "Milvus Dev Test Out directory don't exists!"
+                }
+            }
+        }
+    }
+}

ci/main_jenkinsfile

+pipeline {
+    agent none
+
+    options {
+        timestamps()
+    }
+
+    environment {
+        PROJECT_NAME = "milvus"
+        LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase()
+        SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}"
+        GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}"
+        SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}"
+        DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}"
+    }
+
+    stages {
+        stage("Ubuntu 16.04") {
+            environment {
+                PACKAGE_VERSION = VersionNumber([
+                    versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}'
+                ]);
+
+                DOCKER_VERSION = VersionNumber([
+                    versionNumberString : '${DOCKER_VERSION_STR}'
+                ]);
+            }
+
+            stages {
+                stage("Run Build") {
+                    agent {
+                        kubernetes {
+                            cloud 'build-kubernetes'
+                            label 'build'
+                            defaultContainer 'jnlp'
+                            containerTemplate {
+                                name 'milvus-build-env'
+                                image 'registry.zilliz.com/milvus/milvus-build-env:v0.10'
+                                ttyEnabled true
+                                command 'cat'
+                            }
+                        }
+                    }
+                    stages {
+                        stage('Build') {
+                            steps {
+                                gitlabCommitStatus(name: 'Build') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy"
+                                        load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    post {
+                        aborted {
+                            script {
+                                updateGitlabCommitStatus name: 'Build', state: 'canceled'
+                                echo "Milvus Build aborted !"
+                            }
+                        }
+
+                        failure {
+                            script {
+                                updateGitlabCommitStatus name: 'Build', state: 'failed'
+                                echo "Milvus Build failure !"
+                            }
+                        }
+                    }
+                }
+
+                stage("Publish docker and helm") {
+                    agent {
+                        kubernetes {
+                            label 'publish'
+                            defaultContainer 'jnlp'
+                            yaml """
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: publish
+    componet: docker
+spec:
+  containers:
+  - name: publish-docker
+    image: registry.zilliz.com/library/zilliz_docker:v1.0.0
+    securityContext:
+      privileged: true
+    command:
+    - cat
+    tty: true
+    volumeMounts:
+    - name: docker-sock
+      mountPath: /var/run/docker.sock
+  volumes:
+  - name: docker-sock
+    hostPath:
+      path: /var/run/docker.sock
+"""
+                        }
+                    }
+                    stages {
+                        stage('Publish Docker') {
+                            steps {
+                                gitlabCommitStatus(name: 'Publish Docker') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    post {
+                        aborted {
+                            script {
+                                updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled'
+                                echo "Milvus Publish Docker aborted !"
+                            }
+                        }
+
+                        failure {
+                            script {
+                                updateGitlabCommitStatus name: 'Publish Docker', state: 'failed'
+                                echo "Milvus Publish Docker failure !"
+                            }
+                        }
+                    }
+                }
+
+                stage("Deploy to Development") {
+                    stages {
+                        stage("Deploy to Dev") {
+                            agent {
+                                kubernetes {
+                                    label 'jenkins-slave'
+                                    defaultContainer 'jnlp'
+                                }
+                            }
+                            stages {
+                                stage('Deploy') {
+                                    steps {
+                                        gitlabCommitStatus(name: 'Deloy to Dev') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy"
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            post {
+                                aborted {
+                                    script {
+                                        updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled'
+                                        echo "Milvus Deloy to Dev aborted !"
+                                    }
+                                }
+
+                                failure {
+                                    script {
+                                        updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed'
+                                        echo "Milvus Deloy to Dev failure !"
+                                    }
+                                }
+                            }
+                        }
+
+                        stage("Dev Test") {
+                            agent {
+                                kubernetes {
+                                    label 'test'
+                                    defaultContainer 'jnlp'
+                                    containerTemplate {
+                                        name 'milvus-testframework'
+                                        image 'registry.zilliz.com/milvus/milvus-test:v0.1'
+                                        ttyEnabled true
+                                        command 'cat'
+                                    }
+                                }
+                            }
+                            stages {
+                                stage('Test') {
+                                    steps {
+                                        script {
+                                            load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy"
+                                            load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy"
+                                        }
+                                    }
+                                }
+                            }
+                        }
+
+                        stage ("Cleanup Dev") {
+                            agent {
+                                kubernetes {
+                                    label 'jenkins-slave'
+                                    defaultContainer 'jnlp'
+                                }
+                            }
+                            stages {
+                                stage('Cleanup') {
+                                    steps {
+                                        gitlabCommitStatus(name: 'Cleanup Dev') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            post {
+                                aborted {
+                                    script {
+                                        updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled'
+                                        echo "Milvus Cleanup Dev aborted !"
+                                    }
+                                }
+
+                                failure {
+                                    script {
+                                        updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed'
+                                        echo "Milvus Cleanup Dev failure !"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    post {
+        success {
+            script {
+                updateGitlabCommitStatus name: 'CI/CD', state: 'success'
+                echo "Milvus CI/CD success !"
+            }
+        }
+
+        aborted {
+            script {
+                updateGitlabCommitStatus name: 'CI/CD', state: 'canceled'
+                echo "Milvus CI/CD aborted !"
+            }
+        }
+
+        failure {
+            script {
+                updateGitlabCommitStatus name: 'CI/CD', state: 'failed'
+                echo "Milvus CI/CD failure !"
+            }
+        }
+    }
+}

ci/main_jenkinsfile_no_ut

+pipeline {
+    agent none
+
+    options {
+        timestamps()
+    }
+
+    environment {
+        PROJECT_NAME = "milvus"
+        LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase()
+        SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}"
+        GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}"
+        SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}"
+        DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}"
+    }
+
+    stages {
+        stage("Ubuntu 16.04") {
+            environment {
+                PACKAGE_VERSION = VersionNumber([
+                    versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}'
+                ]);
+
+                DOCKER_VERSION = VersionNumber([
+                    versionNumberString : '${DOCKER_VERSION_STR}'
+                ]);
+            }
+
+            stages {
+                stage("Run Build") {
+                    agent {
+                        kubernetes {
+                            cloud 'build-kubernetes'
+                            label 'build'
+                            defaultContainer 'jnlp'
+                            containerTemplate {
+                                name 'milvus-build-env'
+                                image 'registry.zilliz.com/milvus/milvus-build-env:v0.10'
+                                ttyEnabled true
+                                command 'cat'
+                            }
+                        }
+                    }
+                    stages {
+                        stage('Build') {
+                            steps {
+                                gitlabCommitStatus(name: 'Build') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build_no_ut.groovy"
+                                        load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus_no_ut.groovy"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    post {
+                        aborted {
+                            script {
+                                updateGitlabCommitStatus name: 'Build', state: 'canceled'
+                                echo "Milvus Build aborted !"
+                            }
+                        }
+
+                        failure {
+                            script {
+                                updateGitlabCommitStatus name: 'Build', state: 'failed'
+                                echo "Milvus Build failure !"
+                            }
+                        }
+                    }
+                }
+
+                stage("Publish docker and helm") {
+                    agent {
+                        kubernetes {
+                            label 'publish'
+                            defaultContainer 'jnlp'
+                            yaml """
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: publish
+    componet: docker
+spec:
+  containers:
+  - name: publish-docker
+    image: registry.zilliz.com/library/zilliz_docker:v1.0.0
+    securityContext:
+      privileged: true
+    command:
+    - cat
+    tty: true
+    volumeMounts:
+    - name: docker-sock
+      mountPath: /var/run/docker.sock
+  volumes:
+  - name: docker-sock
+    hostPath:
+      path: /var/run/docker.sock
+"""
+                        }
+                    }
+                    stages {
+                        stage('Publish Docker') {
+                            steps {
+                                gitlabCommitStatus(name: 'Publish Docker') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    post {
+                        aborted {
+                            script {
+                                updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled'
+                                echo "Milvus Publish Docker aborted !"
+                            }
+                        }
+
+                        failure {
+                            script {
+                                updateGitlabCommitStatus name: 'Publish Docker', state: 'failed'
+                                echo "Milvus Publish Docker failure !"
+                            }
+                        }
+                    }
+                }
+
+                stage("Deploy to Development") {
+                    stages {
+                        stage("Deploy to Dev") {
+                            agent {
+                                kubernetes {
+                                    label 'jenkins-slave'
+                                    defaultContainer 'jnlp'
+                                }
+                            }
+                            stages {
+                                stage('Deploy') {
+                                    steps {
+                                        gitlabCommitStatus(name: 'Deloy to Dev') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy"
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            post {
+                                aborted {
+                                    script {
+                                        updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled'
+                                        echo "Milvus Deloy to Dev aborted !"
+                                    }
+                                }
+
+                                failure {
+                                    script {
+                                        updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed'
+                                        echo "Milvus Deloy to Dev failure !"
+                                    }
+                                }
+                            }
+                        }
+
+                        stage("Dev Test") {
+                            agent {
+                                kubernetes {
+                                    label 'test'
+                                    defaultContainer 'jnlp'
+                                    containerTemplate {
+                                        name 'milvus-testframework'
+                                        image 'registry.zilliz.com/milvus/milvus-test:v0.1'
+                                        ttyEnabled true
+                                        command 'cat'
+                                    }
+                                }
+                            }
+                            stages {
+                                stage('Test') {
+                                    steps {
+                                        script {
+                                            load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy"
+                                            load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy"
+                                        }
+                                    }
+                                }
+                            }
+                        }
+
+                        stage ("Cleanup Dev") {
+                            agent {
+                                kubernetes {
+                                    label 'jenkins-slave'
+                                    defaultContainer 'jnlp'
+                                }
+                            }
+                            stages {
+                                stage('Cleanup') {
+                                    steps {
+                                        gitlabCommitStatus(name: 'Cleanup Dev') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            post {
+                                aborted {
+                                    script {
+                                        updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled'
+                                        echo "Milvus Cleanup Dev aborted !"
+                                    }
+                                }
+
+                                failure {
+                                    script {
+                                        updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed'
+                                        echo "Milvus Cleanup Dev failure !"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    post {
+        success {
+            script {
+                updateGitlabCommitStatus name: 'CI/CD', state: 'success'
+                echo "Milvus CI/CD success !"
+            }
+        }
+
+        aborted {
+            script {
+                updateGitlabCommitStatus name: 'CI/CD', state: 'canceled'
+                echo "Milvus CI/CD aborted !"
+            }
+        }
+
+        failure {
+            script {
+                updateGitlabCommitStatus name: 'CI/CD', state: 'failed'
+                echo "Milvus CI/CD failure !"
+            }
+        }
+    }
+}

ci/pod_containers/milvus-engine-build.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus
+    componet: build-env
+spec:
+  containers:
+  - name: milvus-build-env
+    image: registry.zilliz.com/milvus/milvus-build-env:v0.9
+    command:
+    - cat
+    tty: true

ci/pod_containers/milvus-testframework.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus
+    componet: testframework
+spec:
+  containers:
+  - name: milvus-testframework
+    image: registry.zilliz.com/milvus/milvus-test:v0.1
+    command:
+    - cat
+    tty: true

ci/pod_containers/publish-docker.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: publish
+    componet: docker
+spec:
+  containers:
+  - name: publish-docker
+    image: registry.zilliz.com/library/zilliz_docker:v1.0.0
+    securityContext:
+      privileged: true
+    command:
+    - cat
+    tty: true
+    volumeMounts:
+    - name: docker-sock
+      mountPath: /var/run/docker.sock
+  volumes:
+  - name: docker-sock
+    hostPath:
+      path: /var/run/docker.sock