diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy index 056b8fa8c5fd4a2f849a7e4a0d9482d34eec81b5..2e9332fa6e623b07e74837d5e5feabed09bd3ce5 100644 --- a/ci/jenkinsfile/cleanup_dev.groovy +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -1,14 +1,13 @@ try { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - - if (currentBuild.result == 'ABORTED') { - throw new hudson.AbortException("Dev Test Aborted !") - } else if (currentBuild.result == 'FAILURE') { - error("Dev Test Failure !") + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" } } catch (exc) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } throw exc } diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy index dc18067cdda256d6bcaf9e0279097e9d71984dff..e57988fefe66ca0bb58891874a1e248cf6e0aac1 100644 --- a/ci/jenkinsfile/cluster_cleanup_dev.groovy +++ b/ci/jenkinsfile/cluster_cleanup_dev.groovy @@ -1,14 +1,13 @@ try { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - - if (currentBuild.result == 'ABORTED') { - throw new hudson.AbortException("Cluster Dev Test Aborted !") - } else if (currentBuild.result == 'FAILURE') { - error("Dev Test Failure !") + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" } } catch (exc) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" + } throw exc } diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy index dba2e52713237b19523444289e71395bcf32174b..cde361b9eac401fecfbc3a8a7dd0541b5072fad9 100644 --- a/ci/jenkinsfile/cluster_deploy2dev.groovy +++ b/ci/jenkinsfile/cluster_deploy2dev.groovy @@ -7,15 +7,14 @@ try { dir ("milvus/milvus-cluster") { sh "helm install --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.3.1 . " } - timeout(time: 2, unit: 'MINUTES') { - waitUntil { - def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true - return !result - } + } + timeout(time: 2, unit: 'MINUTES') { + waitUntil { + def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true + return !result } } } catch (exc) { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' echo 'Helm running failed!' sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" throw exc diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy index 50fd67f80392e9301d27aacc17c3bfd487b794df..8093f67d181fac5b8e220d0ccf26b41ef32f1d5a 100644 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -1,18 +1,12 @@ -container('milvus-testframework') { - timeout(time: 10, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Dev Test') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Dev Test', state: 'failed' - currentBuild.result = 'FAILURE' - echo 'Milvus Test Failed !' - } +timeout(time: 10, unit: 'MINUTES') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements.txt' + sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" } + } catch (exc) { + echo 'Milvus Test Failed !' + throw exc } } - diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index d20f1444bc98bc8d8947274719c3bc33e240138a..b0c537f2e1462bd849c1d8f03e169a736a467f52 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -10,12 +10,11 @@ try { } timeout(time: 2, unit: 'MINUTES') { waitUntil { - def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true + def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true return !result } } } catch (exc) { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' echo 'Helm running failed!' sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" throw exc diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index 13e1c868b8620ac26243349a8fc6908e1466cebd..836ddbad051bcea4ce740e9ae8d3787f10998164 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -1,18 +1,12 @@ -container('milvus-testframework') { - timeout(time: 10, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Dev Test') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Dev Test', state: 'failed' - currentBuild.result = 'FAILURE' - echo 'Milvus Test Failed !' - } +timeout(time: 10, unit: 'MINUTES') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements.txt' + sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" } + } catch (exc) { + echo 'Milvus Test Failed !' + throw exc } } - diff --git a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy index 52fecc6c1dcb71214d5bdb40b9ab9af72ee18c65..6bbd8a649fe3de9f1b7c84173a01e6db66099828 100644 --- a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy +++ b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy @@ -1,26 +1,14 @@ -container('milvus-testframework') { - timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - gitlabCommitStatus(name: 'Upload Dev Test Out') { - if (fileExists('cluster_test_out')) { - try { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("cluster_test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } catch (hudson.AbortException ae) { - updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'canceled' - currentBuild.result = 'ABORTED' - } catch (exc) { - updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed' - currentBuild.result = 'FAILURE' - } - } else { - updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed' - echo "Milvus Dev Test Out directory don't exists!" - } +timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + if (fileExists('cluster_test_out')) { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("cluster_test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" } + } else { + error("Milvus Dev Test Out directory don't exists!") } } } + diff --git a/ci/jenkinsfile/upload_dev_test_out.groovy b/ci/jenkinsfile/upload_dev_test_out.groovy index c401b16608e81b343d24f93a7130b151b676e97d..7e106c0296a52c3330d29d933cea4b8f91ced5c5 100644 --- a/ci/jenkinsfile/upload_dev_test_out.groovy +++ b/ci/jenkinsfile/upload_dev_test_out.groovy @@ -1,26 +1,13 @@ -container('milvus-testframework') { - timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - gitlabCommitStatus(name: 'Upload Dev Test Out') { - if (fileExists('test_out')) { - try { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } catch (hudson.AbortException ae) { - updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'canceled' - currentBuild.result = 'ABORTED' - } catch (exc) { - updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed' - currentBuild.result = 'FAILURE' - } - } else { - updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed' - echo "Milvus Dev Test Out directory don't exists!" - } +timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + if (fileExists('test_out')) { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" } + } else { + error("Milvus Dev Test Out directory don't exists!") } } } diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index a324ff44416c139d27b98af9351e85d5e2a7a2bf..9577837bfd98af680d408788a500eb964c783daa 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -131,59 +131,53 @@ spec: stage("Deploy to Development") { parallel { - stage("Single") { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + stages { stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" - } - } - } } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { script { load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" @@ -192,140 +186,133 @@ spec: } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" - } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } + success { + script { + echo "Milvus Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Single Node CI/CD failure !" + } + } } } stage("Cluster") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } stages { stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" - } - } - } } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { script { load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_cluster_dev_test_out.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" } } } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" - } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } + success { + script { + echo "Milvus Cluster CI/CD success !" + } + } + aborted { + script { + echo "Milvus Cluster CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Cluster CI/CD failure !" + } + } } } } diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 203b185e146875bf20dc7575bf32c1e0bfbd09fd..082433650c2ed70bc4a27210d08ea87c7e01566e 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -131,59 +131,53 @@ spec: stage("Deploy to Development") { parallel { - stage("Single") { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + stages { stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" - } - } - } } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { script { load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" @@ -192,140 +186,133 @@ spec: } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" - } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } + success { + script { + echo "Milvus Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Single Node CI/CD failure !" + } + } } } stage("Cluster") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } stages { stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" - } - } - } } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { script { load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_cluster_dev_test_out.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" } } } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" - } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } + success { + script { + echo "Milvus Cluster CI/CD success !" + } + } + aborted { + script { + echo "Milvus Cluster CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Cluster CI/CD failure !" + } + } } } } @@ -337,16 +324,18 @@ spec: post { always { script { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' + if (env.gitlabAfter != null) { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } } } } diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index c375cc87eae59874892312f1ff6f2ba2ebfad385..7f59561f82d61dd076164db49cd87940d5ca6d5b 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -131,59 +131,53 @@ spec: stage("Deploy to Development") { parallel { - stage("Single") { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + stages { stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" - } - } - } } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { script { load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" @@ -192,140 +186,133 @@ spec: } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" - } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } + success { + script { + echo "Milvus Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Single Node CI/CD failure !" + } + } } } stage("Cluster") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } stages { stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" - } - } - } } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { script { load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_cluster_dev_test_out.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" } } } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" - } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } + success { + script { + echo "Milvus Cluster CI/CD success !" + } + } + aborted { + script { + echo "Milvus Cluster CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Cluster CI/CD failure !" + } + } } } } diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 8e39f7fbb2e4b5ba5f653a63e024cbeae491d8da..0d17ecbb1604d642753b568cb2882840b75a0762 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -55,15 +55,13 @@ TEST_F(DBTest, CONFIG_TEST) { engine::ArchiveConf conf("delete"); ASSERT_EQ(conf.GetType(), "delete"); auto criterias = conf.GetCriterias(); - ASSERT_TRUE(criterias.size() == 1); - ASSERT_TRUE(criterias["disk"] == 512); + ASSERT_TRUE(criterias.size() == 0); } { engine::ArchiveConf conf("swap"); ASSERT_EQ(conf.GetType(), "swap"); auto criterias = conf.GetCriterias(); - ASSERT_TRUE(criterias.size() == 1); - ASSERT_TRUE(criterias["disk"] == 512); + ASSERT_TRUE(criterias.size() == 0); } { ASSERT_ANY_THROW(engine::ArchiveConf conf1("swap", "disk:")); @@ -209,11 +207,21 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_STATS(stat); } - sleep(2); // wait until build index finish + db_->BuildIndex(TABLE_NAME); // wait until build index finish - engine::QueryResults results; - stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); - ASSERT_STATS(stat); + { + engine::QueryResults results; + stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); + ASSERT_STATS(stat); + } + + {//search by specify index file + engine::meta::DatesT dates; + std::vector file_ids = {"1", "2", "3", "4"}; + engine::QueryResults results; + stat = db_->Query(TABLE_NAME, file_ids, k, nq, xq.data(), dates, results); + ASSERT_STATS(stat); + } // TODO(linxj): add groundTruth assert }; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index 4356746fc2a42a325371ea639bbda39622486ea8..9dd07fac8d89b55e8db1099c08bcbe0ff099802b 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -14,6 +14,7 @@ #include "db/Options.h" #include "db/DBMetaImpl.h" #include "db/EngineFactory.h" +#include "db/Utils.h" #include @@ -134,4 +135,32 @@ TEST(DBMiscTest, META_TEST) { int delta = 10; engine::meta::DateT dt = impl.GetDate(tt, delta); ASSERT_GT(dt, 0); +} + +TEST(DBMiscTest, UTILS_TEST) { + engine::DBMetaOptions options; + options.path = "/tmp/milvus_test/main"; + options.slave_paths.push_back("/tmp/milvus_test/slave_1"); + options.slave_paths.push_back("/tmp/milvus_test/slave_2"); + + const std::string TABLE_NAME = "test_tbl"; + auto status = engine::utils::CreateTablePath(options, TABLE_NAME); + ASSERT_TRUE(status.ok()); + ASSERT_TRUE(boost::filesystem::exists(options.path)); + for(auto& path : options.slave_paths) { + ASSERT_TRUE(boost::filesystem::exists(path)); + } + + engine::meta::TableFileSchema file; + file.id_ = 50; + file.table_id_ = TABLE_NAME; + file.file_type_ = 3; + file.date_ = 155000; + status = engine::utils::GetTableFilePath(options, file); + ASSERT_FALSE(status.ok()); + ASSERT_TRUE(file.location_.empty()); + + status = engine::utils::DeleteTablePath(options, TABLE_NAME); + ASSERT_TRUE(status.ok()); + } \ No newline at end of file