Simplify testing Kubernetes application deployment within CI pipeline by using Helm

Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

.jenkins/modules/Regression/CleanupPythonRegression.groovy

+container('deploy-env') {
+    def helmStatus = sh script: "helm status -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}", returnStatus: true
+    if (!helmStatus) {
+        sh "helm uninstall -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}"
+        def labels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME}"
+        sh "kubectl delete pvc -n ${env.HELM_RELEASE_NAMESPACE} \$(kubectl get pvc -n ${env.HELM_RELEASE_NAMESPACE} -l ${labels} -o jsonpath='{range.items[*]}{.metadata.name} ')"
+    }
+}

.jenkins/modules/Regression/PythonRegression.groovy

 timeout(time: 60, unit: 'MINUTES') {
-    try {
-        if ("${REGRESSION_SERVICE_NAME}" == "regression_distributed") {
-            sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d pulsar'
-        }
-        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d etcd'
-        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d minio'
-        dir ('build/docker/deploy') {
-            sh 'docker pull ${TARGET_REPO}/milvus-distributed:${TARGET_TAG}'
-            if ("${REGRESSION_SERVICE_NAME}" == "regression_distributed") {
-                sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d master'
-                sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d indexservice'
-                sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d indexnode'
-                sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d proxyservice'
-                sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d dataservice'
-                sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d queryservice'
-                sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e DATA_NODE_ID=3 -d datanode'
-                sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d proxynode'
-                sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e QUERY_NODE_ID=1 -d querynode'
-                sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e QUERY_NODE_ID=2 -d querynode'
-            } else {
-                sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d standalone'
+    container('deploy-env') {
+        dir ('milvus-helm-chart') {
+            sh " helm version && \
+                 helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts && \
+                 helm repo add bitnami https://charts.bitnami.com/bitnami && \
+                 helm repo add minio https://helm.min.io/ && \
+                 helm repo update"
+
+            def milvusHelmURL = "https://github.com/zilliztech/milvus-helm-charts.git"
+            checkout([$class: 'GitSCM', branches: [[name: "${env.HELM_BRANCH}"]], userRemoteConfigs: [[url: "${milvusHelmURL}"]]])
+
+            dir ('charts/milvus-ha') {
+                sh script: "kubectl create namespace ${env.HELM_RELEASE_NAMESPACE}", returnStatus: true
+
+                def helmCMD = ""
+                if ("${REGRESSION_SERVICE_TYPE}" == "distributed") {
+                    helmCMD = "helm install --wait --timeout 300s \
+                                   --set standalone.enabled=false \
+                                   --set image.all.repository=${env.TARGET_REPO}/milvus-distributed \
+                                   --set image.all.tag=${env.TARGET_TAG} \
+                                   --set image.all.pullPolicy=Always \
+                                   --set logsPersistence.enabled=true \
+                                   --set logsPersistence.mountPath=/milvus-distributed/logs \
+                                   --namespace ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} ."
+                } else {
+                    helmCMD = "helm install --wait --timeout 300s \
+                                   --set image.all.repository=${env.TARGET_REPO}/milvus-distributed \
+                                   --set image.all.tag=${env.TARGET_TAG} \
+                                   --set image.all.pullPolicy=Always \
+                                   --set logsPersistence.enabled=true \
+                                   --set logsPersistence.mountPath=/milvus-distributed/logs \
+                                   --namespace ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} ."
+                }
+
+                try {
+                    sh "${helmCMD}"
+                } catch (exc) {
+                    def helmStatusCMD = "helm get manifest -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} | kubectl describe -n ${env.HELM_RELEASE_NAMESPACE} -f - && \
+                                         helm status -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}"
+                    sh script: helmStatusCMD, returnStatus: true
+                    throw exc
+                }
             }
         }
+    }
 
-        dir ('build/docker/test') {
-            sh 'docker pull ${SOURCE_REPO}/pytest:${SOURCE_TAG} || true'
-            sh 'docker-compose build --force-rm ${REGRESSION_SERVICE_NAME}'
-            sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run --rm ${REGRESSION_SERVICE_NAME}'
-            try {
-                withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
-                    sh 'docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${DOKCER_REGISTRY_URL}'
-                    sh 'docker-compose push ${REGRESSION_SERVICE_NAME} || true'
+    container('test-env') {
+        try {
+            dir ('tests/python_test') {
+                sh "pytest --tags=0331 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local"
+            }
+        } catch (exc) {
+            echo 'PyTest Regression Failed !'
+            throw exc
+        } finally {
+            container('deploy-env') {
+                def labels = ""
+                if ("${REGRESSION_SERVICE_TYPE}" == "distributed") {
+                    labels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},component=proxyservice"
+                } else {
+                    labels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},component=standalone"
                 }
-            } catch (exc) {
-                throw exc
-            } finally {
-                sh 'docker logout ${DOKCER_REGISTRY_URL}'
+                sh "mkdir -p ${env.DEV_TEST_ARTIFACTS_PATH}"
+                sh "kubectl cp -n ${env.HELM_RELEASE_NAMESPACE} \$(kubectl get pod -n ${env.HELM_RELEASE_NAMESPACE} -l ${labels} -o jsonpath='{range.items[0]}{.metadata.name}'):logs ${env.DEV_TEST_ARTIFACTS_PATH}"
+                archiveArtifacts artifacts: "${env.DEV_TEST_ARTIFACTS_PATH}/**", allowEmptyArchive: true
             }
         }
-    } catch(exc) {
-        throw exc
-    } finally {
-        dir ('build/docker/deploy') {
-            sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} ps -a | tail -n +3 | awk \'{ print $1 }\' | ( while read arg; do docker logs -t $arg > $arg.log 2>&1; done )'
-            archiveArtifacts artifacts: "**.log", allowEmptyArchive: true
-            sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} down --rmi all -v || true'
-        }
-        if ("${REGRESSION_SERVICE_NAME}" == "regression_distributed") {
-            sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v pulsar'
-        }
-        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v etcd'
-        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v minio'
-        dir ('build/docker/test') {
-            sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run --rm ${REGRESSION_SERVICE_NAME} /bin/bash -c "find . -name \'__pycache__\' | xargs rm -rf && find . -name \'.pytest_cache\' | xargs rm -rf"'
-            sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} down --rmi all -v || true'
-        }
     }
 }

build/ci/jenkins/Jenkinsfile

@@ -7,7 +7,7 @@ pipeline {
         timestamps()
     }
     environment {
-        PROJECT_NAME = "milvus-distributed"
+        PROJECT_NAME = "milvus"
         SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
         BUILD_TYPE = "Release"
         LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase()
@@ -71,15 +71,22 @@ pipeline {
             matrix {
                 axes {
                     axis {
-                        name 'REGRESSION_SERVICE_NAME'
-                        values 'regression_standalone', 'regression_distributed'
+                        name 'REGRESSION_SERVICE_TYPE'
+                        values 'standalone', 'distributed'
                     }
                 }
                 environment {
-                    DOCKER_COMPOSE_PROJECT_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}".replaceAll("\\.", "-").replaceAll("_", "-")
+                    HELM_BRANCH = "main"
+                    HELM_RELEASE_NAMESPACE = "dev"
+                    HELM_RELEASE_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-${REGRESSION_SERVICE_TYPE}".replaceAll("\\.", "-").replaceAll("_", "-").toLowerCase()
+                    DEV_TEST_ARTIFACTS_PATH = "artifacts"
                 }
                 agent {
-                    label 'performance'
+                    kubernetes {
+                        label "${env.PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-${REGRESSION_SERVICE_TYPE}-devtest"
+                        defaultContainer 'test-env'
+                        yamlFile "build/ci/jenkins/pod/test-env.yaml"
+                    }
                 }
                 stages {
                     stage('Test') {
@@ -90,7 +97,7 @@ pipeline {
                 }
                 post {
                     cleanup {
-                        deleteDir() /* clean up our workspace */
+                        MPLModule('Cleanup Python Regression')
                     }
                 }
             }

build/ci/jenkins/pod/test-env.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus
+    componet: test-env
+spec:
+  containers:
+  - name: test-env
+    image: milvusdb/pytest:20210310-5ea22d7
+    command:
+    - cat
+    tty: true
+    resources:
+      limits:
+        memory: "8Gi"
+        cpu: "4.0"
+      requests:
+        memory: "4Gi"
+        cpu: "2.0"
+  - name: deploy-env
+    image: registry.zilliz.com/library/helm3:v0.1
+    command:
+    - cat
+    tty: true
+    volumeMounts:
+    - name: kubeconf
+      mountPath: /root/.kube/
+      readOnly: true
+  volumes:
+  - name: kubeconf
+    secret:
+      secretName: test-cluster-config
+  tolerations:
+  - key: dedicated
+    operator: Equal
+    value: milvus
+    effect: NoSchedule

build/docker/test/.env

 SOURCE_REPO=milvusdb
 TARGET_REPO=milvusdb
-SOURCE_TAG=latest
-TARGET_TAG=latest
\ No newline at end of file
+SOURCE_TAG=20210310-5ea22d7
+TARGET_TAG=20210310-5ea22d7
\ No newline at end of file