Merge pull request #99 from milvus-ci-robot/0.5.1

Add ELK for Milvus 0.5.1 Jenkins CI

Former-commit-id: 19f87bc123aaf119e7b507cce73c77e71f79a323

ci/jenkins/Jenkinsfile

+String cron_string = BRANCH_NAME == "master" || BRANCH_NAME == "0.5.0" || BRANCH_NAME == "0.5.1" ? "H 0 * * *" : ""
 pipeline {
     agent none
 
+    triggers { cron(cron_string) }
+
     options {
         timestamps()
     }
@@ -20,6 +23,7 @@ pipeline {
         SEMVER = "${BRANCH_NAME}"
         JOBNAMES = env.JOB_NAME.split('/')
         PIPELINE_NAME = "${JOBNAMES[0]}"
+        NIGHTLIY_TEST = "${cron_string == "" ? false : true}"
     }
 
     stages {
@@ -119,7 +123,11 @@ pipeline {
                             steps {
                                 container('milvus-test-env') {
                                     script {
-                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/singleDevTest.groovy"
+                                        if (NIGHTLIY_TEST) {
+                                            load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/singleDevNightlyTest.groovy"
+                                        } else {
+                                            load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/singleDevTest.groovy"
+                                        }
                                     }
                                 }
                             }

ci/jenkins/jenkinsfile/deploySingle2Dev.groovy

@@ -4,7 +4,7 @@ try {
     dir ('milvus-helm') {
         checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
         dir ("milvus-gpu") {
-            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/values.yaml --namespace milvus ."
+            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
         }
     }
 } catch (exc) {

ci/jenkins/jenkinsfile/singleDevNightlyTest.groovy

+timeout(time: 90, unit: 'MINUTES') {
+    dir ("tests/milvus_python_test") {
+        sh 'python3 -m pip install -r requirements.txt'
+        sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local"
+    }
+    // mysql database backend test
+    load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
+
+    if (!fileExists('milvus-helm')) {
+        dir ("milvus-helm") {
+            checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
+        }
+    }
+    dir ("milvus-helm") {
+        dir ("milvus-gpu") {
+            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
+        }
+    }
+    dir ("tests/milvus_python_test") {
+        sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local"
+    }
+}

ci/jenkins/jenkinsfile/singleDevTest.groovy

@@ -13,7 +13,7 @@ timeout(time: 60, unit: 'MINUTES') {
     }
     dir ("milvus-helm") {
         dir ("milvus-gpu") {
-            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml --namespace milvus ."
+            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
         }
     }
     dir ("tests/milvus_python_test") {