From f39eb065461fc9640409978af9c86f60b3361b34 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 22 Nov 2019 15:01:57 +0800 Subject: [PATCH] clean up old jenkins ci --- ci/function/file_transfer.groovy | 10 - ci/jenkinsfile/cleanup_dev.groovy | 13 - ci/jenkinsfile/cleanup_staging.groovy | 13 - ci/jenkinsfile/cluster_cleanup_dev.groovy | 13 - ci/jenkinsfile/cluster_deploy2dev.groovy | 24 - ci/jenkinsfile/cluster_dev_test.groovy | 12 - ci/jenkinsfile/deploy2dev.groovy | 16 - ci/jenkinsfile/deploy2staging.groovy | 16 - ci/jenkinsfile/dev_test.groovy | 28 - ci/jenkinsfile/dev_test_all.groovy | 29 -- ci/jenkinsfile/milvus_build.groovy | 30 -- ci/jenkinsfile/milvus_build_no_ut.groovy | 28 - ci/jenkinsfile/nightly_publish_docker.groovy | 38 -- ci/jenkinsfile/notify.groovy | 15 - ci/jenkinsfile/packaged_milvus.groovy | 44 -- ci/jenkinsfile/packaged_milvus_no_ut.groovy | 26 - ci/jenkinsfile/publish_docker.groovy | 35 -- ci/jenkinsfile/staging_test.groovy | 31 -- .../upload_dev_cluster_test_out.groovy | 14 - ci/jenkinsfile/upload_dev_test_out.groovy | 13 - ci/jenkinsfile/upload_staging_test_out.groovy | 13 - ci/main_jenkinsfile | 396 --------------- ci/main_jenkinsfile_no_ut | 396 --------------- ci/nightly_main_jenkinsfile | 478 ------------------ ci/pod_containers/milvus-engine-build.yaml | 13 - ci/pod_containers/milvus-testframework.yaml | 13 - ci/pod_containers/publish-docker.yaml | 22 - 27 files changed, 1779 deletions(-) delete mode 100644 ci/function/file_transfer.groovy delete mode 100644 ci/jenkinsfile/cleanup_dev.groovy delete mode 100644 ci/jenkinsfile/cleanup_staging.groovy delete mode 100644 ci/jenkinsfile/cluster_cleanup_dev.groovy delete mode 100644 ci/jenkinsfile/cluster_deploy2dev.groovy delete mode 100644 ci/jenkinsfile/cluster_dev_test.groovy delete mode 100644 ci/jenkinsfile/deploy2dev.groovy delete mode 100644 ci/jenkinsfile/deploy2staging.groovy delete mode 100644 ci/jenkinsfile/dev_test.groovy delete mode 100644 ci/jenkinsfile/dev_test_all.groovy delete mode 100644 ci/jenkinsfile/milvus_build.groovy delete mode 100644 ci/jenkinsfile/milvus_build_no_ut.groovy delete mode 100644 ci/jenkinsfile/nightly_publish_docker.groovy delete mode 100644 ci/jenkinsfile/notify.groovy delete mode 100644 ci/jenkinsfile/packaged_milvus.groovy delete mode 100644 ci/jenkinsfile/packaged_milvus_no_ut.groovy delete mode 100644 ci/jenkinsfile/publish_docker.groovy delete mode 100644 ci/jenkinsfile/staging_test.groovy delete mode 100644 ci/jenkinsfile/upload_dev_cluster_test_out.groovy delete mode 100644 ci/jenkinsfile/upload_dev_test_out.groovy delete mode 100644 ci/jenkinsfile/upload_staging_test_out.groovy delete mode 100644 ci/main_jenkinsfile delete mode 100644 ci/main_jenkinsfile_no_ut delete mode 100644 ci/nightly_main_jenkinsfile delete mode 100644 ci/pod_containers/milvus-engine-build.yaml delete mode 100644 ci/pod_containers/milvus-testframework.yaml delete mode 100644 ci/pod_containers/publish-docker.yaml diff --git a/ci/function/file_transfer.groovy b/ci/function/file_transfer.groovy deleted file mode 100644 index bebae148..00000000 --- a/ci/function/file_transfer.groovy +++ /dev/null @@ -1,10 +0,0 @@ -def FileTransfer (sourceFiles, remoteDirectory, remoteIP, protocol = "ftp", makeEmptyDirs = true) { - if (protocol == "ftp") { - ftpPublisher masterNodeName: '', paramPublish: [parameterName: ''], alwaysPublishFromMaster: false, continueOnError: false, failOnError: true, publishers: [ - [configName: "${remoteIP}", transfers: [ - [asciiMode: false, cleanRemote: false, excludes: '', flatten: false, makeEmptyDirs: "${makeEmptyDirs}", noDefaultExcludes: false, patternSeparator: '[, ]+', remoteDirectory: "${remoteDirectory}", remoteDirectorySDF: false, removePrefix: '', sourceFiles: "${sourceFiles}"]], usePromotionTimestamp: true, useWorkspaceInPromotion: false, verbose: true - ] - ] - } -} -return this diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy deleted file mode 100644 index 2e9332fa..00000000 --- a/ci/jenkinsfile/cleanup_dev.groovy +++ /dev/null @@ -1,13 +0,0 @@ -try { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - } -} catch (exc) { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - } - throw exc -} - diff --git a/ci/jenkinsfile/cleanup_staging.groovy b/ci/jenkinsfile/cleanup_staging.groovy deleted file mode 100644 index 2e9332fa..00000000 --- a/ci/jenkinsfile/cleanup_staging.groovy +++ /dev/null @@ -1,13 +0,0 @@ -try { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - } -} catch (exc) { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - } - throw exc -} - diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy deleted file mode 100644 index e57988fe..00000000 --- a/ci/jenkinsfile/cluster_cleanup_dev.groovy +++ /dev/null @@ -1,13 +0,0 @@ -try { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - } -} catch (exc) { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - } - throw exc -} - diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy deleted file mode 100644 index 7f2a5842..00000000 --- a/ci/jenkinsfile/cluster_deploy2dev.groovy +++ /dev/null @@ -1,24 +0,0 @@ -try { - sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' - sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' - sh 'helm repo update' - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("milvus/milvus-cluster") { - sh "helm install --wait --timeout 300 --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.5.0 . " - } - } - /* - timeout(time: 2, unit: 'MINUTES') { - waitUntil { - def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true - return !result - } - } - */ -} catch (exc) { - echo 'Helm running failed!' - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - throw exc -} - diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy deleted file mode 100644 index 4a15b926..00000000 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ /dev/null @@ -1,12 +0,0 @@ -timeout(time: 25, unit: 'MINUTES') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements_cluster.txt' - sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" - } - } catch (exc) { - echo 'Milvus Cluster Test Failed !' - throw exc - } -} diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy deleted file mode 100644 index b00c8fa3..00000000 --- a/ci/jenkinsfile/deploy2dev.groovy +++ /dev/null @@ -1,16 +0,0 @@ -try { - sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' - sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' - sh 'helm repo update' - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ." - } - } -} catch (exc) { - echo 'Helm running failed!' - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - throw exc -} - diff --git a/ci/jenkinsfile/deploy2staging.groovy b/ci/jenkinsfile/deploy2staging.groovy deleted file mode 100644 index 42ccfda7..00000000 --- a/ci/jenkinsfile/deploy2staging.groovy +++ /dev/null @@ -1,16 +0,0 @@ -try { - sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' - sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' - sh 'helm repo update' - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ." - } - } -} catch (exc) { - echo 'Helm running failed!' - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - throw exc -} - diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy deleted file mode 100644 index 9b265ac4..00000000 --- a/ci/jenkinsfile/dev_test.groovy +++ /dev/null @@ -1,28 +0,0 @@ -timeout(time: 30, unit: 'MINUTES') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal=true" - } - // mysql database backend test - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - - if (!fileExists('milvus-helm')) { - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - } - } - dir ("milvus-helm") { - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ." - } - } - dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal=true" - } - } catch (exc) { - echo 'Milvus Test Failed !' - throw exc - } -} diff --git a/ci/jenkinsfile/dev_test_all.groovy b/ci/jenkinsfile/dev_test_all.groovy deleted file mode 100644 index b82d995d..00000000 --- a/ci/jenkinsfile/dev_test_all.groovy +++ /dev/null @@ -1,29 +0,0 @@ -timeout(time: 60, unit: 'MINUTES') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal=true" - } - - // mysql database backend test - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - - if (!fileExists('milvus-helm')) { - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - } - } - dir ("milvus-helm") { - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ." - } - } - dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal=true" - } - } catch (exc) { - echo 'Milvus Test Failed !' - throw exc - } -} diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy deleted file mode 100644 index 27e50c64..00000000 --- a/ci/jenkinsfile/milvus_build.groovy +++ /dev/null @@ -1,30 +0,0 @@ -container('milvus-build-env') { - timeout(time: 120, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Build Engine') { - dir ("milvus_engine") { - try { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - - dir ("core") { - sh "git config --global user.email \"test@zilliz.com\"" - sh "git config --global user.name \"test\"" - withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "./build.sh -l" - sh "rm -rf cmake_build" - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \ - && export JFROG_USER_NAME='${USERNAME}' \ - && export JFROG_PASSWORD='${PASSWORD}' \ - && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \ - && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u -c" - - sh "./coverage.sh -u root -p 123456 -t \$POD_IP" - } - } - } catch (exc) { - updateGitlabCommitStatus name: 'Build Engine', state: 'failed' - throw exc - } - } - } - } -} diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy deleted file mode 100644 index 3f221b89..00000000 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ /dev/null @@ -1,28 +0,0 @@ -container('milvus-build-env') { - timeout(time: 120, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Build Engine') { - dir ("milvus_engine") { - try { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - - dir ("core") { - sh "git config --global user.email \"test@zilliz.com\"" - sh "git config --global user.name \"test\"" - withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "./build.sh -l" - sh "rm -rf cmake_build" - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \ - && export JFROG_USER_NAME='${USERNAME}' \ - && export JFROG_PASSWORD='${PASSWORD}' \ - && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \ - && ./build.sh -t ${params.BUILD_TYPE} -j -d /opt/milvus" - } - } - } catch (exc) { - updateGitlabCommitStatus name: 'Build Engine', state: 'failed' - throw exc - } - } - } - } -} diff --git a/ci/jenkinsfile/nightly_publish_docker.groovy b/ci/jenkinsfile/nightly_publish_docker.groovy deleted file mode 100644 index 8c6121be..00000000 --- a/ci/jenkinsfile/nightly_publish_docker.groovy +++ /dev/null @@ -1,38 +0,0 @@ -container('publish-docker') { - timeout(time: 15, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Publish Engine Docker') { - try { - dir ("${PROJECT_NAME}_build") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("docker/deploy/ubuntu16.04/free_version") { - sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" - sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" - try { - def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}") - docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_PUBLISH_USER}") { - customImage.push() - } - docker.withRegistry('https://zilliz.azurecr.cn', "${params.AZURE_DOCKER_PUBLISH_USER}") { - customImage.push() - } - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'success' - echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled' - throw exc - } finally { - sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}" - } - } - } - } catch (exc) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed' - echo 'Publish docker failed!' - throw exc - } - } - } -} - diff --git a/ci/jenkinsfile/notify.groovy b/ci/jenkinsfile/notify.groovy deleted file mode 100644 index 0a257b8c..00000000 --- a/ci/jenkinsfile/notify.groovy +++ /dev/null @@ -1,15 +0,0 @@ -def notify() { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } -} -return this - diff --git a/ci/jenkinsfile/packaged_milvus.groovy b/ci/jenkinsfile/packaged_milvus.groovy deleted file mode 100644 index 1d30e219..00000000 --- a/ci/jenkinsfile/packaged_milvus.groovy +++ /dev/null @@ -1,44 +0,0 @@ -container('milvus-build-env') { - timeout(time: 5, unit: 'MINUTES') { - dir ("milvus_engine") { - dir ("core") { - gitlabCommitStatus(name: 'Packaged Engine') { - if (fileExists('milvus')) { - try { - sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\"" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' - throw exc - } - } else { - updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' - error("Milvus binary directory don't exists!") - } - } - - gitlabCommitStatus(name: 'Packaged Engine lcov') { - if (fileExists('lcov_out')) { - try { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("lcov_out/", "${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus lcov out Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}/lcov_out/\"" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed' - throw exc - } - } else { - updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed' - error("Milvus lcov out directory don't exists!") - } - } - } - } - } -} diff --git a/ci/jenkinsfile/packaged_milvus_no_ut.groovy b/ci/jenkinsfile/packaged_milvus_no_ut.groovy deleted file mode 100644 index bc68be37..00000000 --- a/ci/jenkinsfile/packaged_milvus_no_ut.groovy +++ /dev/null @@ -1,26 +0,0 @@ -container('milvus-build-env') { - timeout(time: 5, unit: 'MINUTES') { - dir ("milvus_engine") { - dir ("core") { - gitlabCommitStatus(name: 'Packaged Engine') { - if (fileExists('milvus')) { - try { - sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\"" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' - throw exc - } - } else { - updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' - error("Milvus binary directory don't exists!") - } - } - } - } - } -} diff --git a/ci/jenkinsfile/publish_docker.groovy b/ci/jenkinsfile/publish_docker.groovy deleted file mode 100644 index ef31eba9..00000000 --- a/ci/jenkinsfile/publish_docker.groovy +++ /dev/null @@ -1,35 +0,0 @@ -container('publish-docker') { - timeout(time: 15, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Publish Engine Docker') { - try { - dir ("${PROJECT_NAME}_build") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("docker/deploy/ubuntu16.04/free_version") { - sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" - sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" - try { - def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}") - docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_PUBLISH_USER}") { - customImage.push() - } - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'success' - echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled' - throw exc - } finally { - sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}" - } - } - } - } catch (exc) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed' - echo 'Publish docker failed!' - throw exc - } - } - } -} - diff --git a/ci/jenkinsfile/staging_test.groovy b/ci/jenkinsfile/staging_test.groovy deleted file mode 100644 index dcf17871..00000000 --- a/ci/jenkinsfile/staging_test.groovy +++ /dev/null @@ -1,31 +0,0 @@ -timeout(time: 40, unit: 'MINUTES') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt' - def service_ip = sh (script: "kubectl get svc --namespace milvus-1 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim() - sh "pytest . --alluredir=\"test_out/staging/single/sqlite\" --ip ${service_ip}" - } - - // mysql database backend test - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" - - if (!fileExists('milvus-helm')) { - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - } - } - dir ("milvus-helm") { - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ." - } - } - dir ("${PROJECT_NAME}_test") { - def service_ip = sh (script: "kubectl get svc --namespace milvus-2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim() - sh "pytest . --alluredir=\"test_out/staging/single/mysql\" --ip ${service_ip}" - } - } catch (exc) { - echo 'Milvus Test Failed !' - throw exc - } -} diff --git a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy deleted file mode 100644 index 6bbd8a64..00000000 --- a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy +++ /dev/null @@ -1,14 +0,0 @@ -timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - if (fileExists('cluster_test_out')) { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("cluster_test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } else { - error("Milvus Dev Test Out directory don't exists!") - } - } -} - diff --git a/ci/jenkinsfile/upload_dev_test_out.groovy b/ci/jenkinsfile/upload_dev_test_out.groovy deleted file mode 100644 index 017b8873..00000000 --- a/ci/jenkinsfile/upload_dev_test_out.groovy +++ /dev/null @@ -1,13 +0,0 @@ -timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - if (fileExists('test_out/dev')) { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("test_out/dev/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } else { - error("Milvus Dev Test Out directory don't exists!") - } - } -} diff --git a/ci/jenkinsfile/upload_staging_test_out.groovy b/ci/jenkinsfile/upload_staging_test_out.groovy deleted file mode 100644 index 1f1e66ab..00000000 --- a/ci/jenkinsfile/upload_staging_test_out.groovy +++ /dev/null @@ -1,13 +0,0 @@ -timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - if (fileExists('test_out/staging')) { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("test_out/staging/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } else { - error("Milvus Dev Test Out directory don't exists!") - } - } -} diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile deleted file mode 100644 index 0c3fc32e..00000000 --- a/ci/main_jenkinsfile +++ /dev/null @@ -1,396 +0,0 @@ -pipeline { - agent none - - options { - timestamps() - } - - environment { - PROJECT_NAME = "milvus" - LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() - SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" - GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" - SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" - DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" - } - - stages { - stage("Ubuntu 16.04") { - environment { - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' - ]); - - DOCKER_VERSION = VersionNumber([ - versionNumberString : '${DOCKER_VERSION_STR}' - ]); - } - - stages { - stage("Run Build") { - agent { - kubernetes { - cloud 'build-kubernetes' - label 'build' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - name: milvus-build-env - labels: - app: milvus - componet: build-env -spec: - containers: - - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.13 - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - command: - - cat - tty: true - resources: - limits: - memory: "28Gi" - cpu: "10.0" - nvidia.com/gpu: 1 - requests: - memory: "14Gi" - cpu: "5.0" - - name: milvus-mysql - image: mysql:5.6 - env: - - name: MYSQL_ROOT_PASSWORD - value: 123456 - ports: - - containerPort: 3306 - name: mysql -""" - } - } - stages { - stage('Build') { - steps { - gitlabCommitStatus(name: 'Build') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Build', state: 'canceled' - echo "Milvus Build aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Build', state: 'failed' - echo "Milvus Build failure !" - } - } - } - } - - stage("Publish docker and helm") { - agent { - kubernetes { - label 'publish' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-docker - image: registry.zilliz.com/library/zilliz_docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -""" - } - } - stages { - stage('Publish Docker') { - steps { - gitlabCommitStatus(name: 'Publish Docker') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' - echo "Milvus Publish Docker aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' - echo "Milvus Publish Docker failure !" - } - } - } - } - - stage("Deploy to Development") { - parallel { - stage("Single Node") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: test-cluster-config -""" - } - } - - stages { - stage("Deploy to Dev") { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } - } - } - } - } - stage("Dev Test") { - steps { - gitlabCommitStatus(name: 'Deloy Test') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" - } - } - } - } - } - stage ("Cleanup Dev") { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - } - } - } - post { - always { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - success { - script { - echo "Milvus Single Node CI/CD success !" - } - } - aborted { - script { - echo "Milvus Single Node CI/CD aborted !" - } - } - failure { - script { - echo "Milvus Single Node CI/CD failure !" - } - } - } - } - -// stage("Cluster") { -// agent { -// kubernetes { -// label 'dev-test' -// defaultContainer 'jnlp' -// yaml """ -// apiVersion: v1 -// kind: Pod -// metadata: -// labels: -// app: milvus -// componet: test -// spec: -// containers: -// - name: milvus-testframework -// image: registry.zilliz.com/milvus/milvus-test:v0.2 -// command: -// - cat -// tty: true -// volumeMounts: -// - name: kubeconf -// mountPath: /root/.kube/ -// readOnly: true -// volumes: -// - name: kubeconf -// secret: -// secretName: test-cluster-config -// """ -// } -// } -// stages { -// stage("Deploy to Dev") { -// steps { -// gitlabCommitStatus(name: 'Deloy to Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" -// } -// } -// } -// } -// } -// stage("Dev Test") { -// steps { -// gitlabCommitStatus(name: 'Deloy Test') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" -// load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" -// } -// } -// } -// } -// } -// stage ("Cleanup Dev") { -// steps { -// gitlabCommitStatus(name: 'Cleanup Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// } -// } -// } -// post { -// always { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// success { -// script { -// echo "Milvus Cluster CI/CD success !" -// } -// } -// aborted { -// script { -// echo "Milvus Cluster CI/CD aborted !" -// } -// } -// failure { -// script { -// echo "Milvus Cluster CI/CD failure !" -// } -// } -// } -// } - } - } - } - } - } - - post { - always { - script { - if (env.gitlabAfter != null) { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } - } - } - } - - success { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'success' - echo "Milvus CI/CD success !" - } - } - - aborted { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' - echo "Milvus CI/CD aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'failed' - echo "Milvus CI/CD failure !" - } - } - } -} - diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut deleted file mode 100644 index e8d7dae7..00000000 --- a/ci/main_jenkinsfile_no_ut +++ /dev/null @@ -1,396 +0,0 @@ -pipeline { - agent none - - options { - timestamps() - } - - environment { - PROJECT_NAME = "milvus" - LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() - SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" - GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" - SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" - DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" - } - - stages { - stage("Ubuntu 16.04") { - environment { - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' - ]); - - DOCKER_VERSION = VersionNumber([ - versionNumberString : '${DOCKER_VERSION_STR}' - ]); - } - - stages { - stage("Run Build") { - agent { - kubernetes { - cloud 'build-kubernetes' - label 'build' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - name: milvus-build-env - labels: - app: milvus - componet: build-env -spec: - containers: - - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.13 - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - command: - - cat - tty: true - resources: - limits: - memory: "28Gi" - cpu: "10.0" - nvidia.com/gpu: 1 - requests: - memory: "14Gi" - cpu: "5.0" - - name: milvus-mysql - image: mysql:5.6 - env: - - name: MYSQL_ROOT_PASSWORD - value: 123456 - ports: - - containerPort: 3306 - name: mysql -""" - } - } - stages { - stage('Build') { - steps { - gitlabCommitStatus(name: 'Build') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build_no_ut.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus_no_ut.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Build', state: 'canceled' - echo "Milvus Build aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Build', state: 'failed' - echo "Milvus Build failure !" - } - } - } - } - - stage("Publish docker and helm") { - agent { - kubernetes { - label 'publish' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-docker - image: registry.zilliz.com/library/zilliz_docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -""" - } - } - stages { - stage('Publish Docker') { - steps { - gitlabCommitStatus(name: 'Publish Docker') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' - echo "Milvus Publish Docker aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' - echo "Milvus Publish Docker failure !" - } - } - } - } - - stage("Deploy to Development") { - parallel { - stage("Single Node") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: test-cluster-config -""" - } - } - - stages { - stage("Deploy to Dev") { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } - } - } - } - } - stage("Dev Test") { - steps { - gitlabCommitStatus(name: 'Deloy Test') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" - } - } - } - } - } - stage ("Cleanup Dev") { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - } - } - } - post { - always { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - success { - script { - echo "Milvus Single Node CI/CD success !" - } - } - aborted { - script { - echo "Milvus Single Node CI/CD aborted !" - } - } - failure { - script { - echo "Milvus Single Node CI/CD failure !" - } - } - } - } - -// stage("Cluster") { -// agent { -// kubernetes { -// label 'dev-test' -// defaultContainer 'jnlp' -// yaml """ -// apiVersion: v1 -// kind: Pod -// metadata: -// labels: -// app: milvus -// componet: test -// spec: -// containers: -// - name: milvus-testframework -// image: registry.zilliz.com/milvus/milvus-test:v0.2 -// command: -// - cat -// tty: true -// volumeMounts: -// - name: kubeconf -// mountPath: /root/.kube/ -// readOnly: true -// volumes: -// - name: kubeconf -// secret: -// secretName: test-cluster-config -// """ -// } -// } -// stages { -// stage("Deploy to Dev") { -// steps { -// gitlabCommitStatus(name: 'Deloy to Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" -// } -// } -// } -// } -// } -// stage("Dev Test") { -// steps { -// gitlabCommitStatus(name: 'Deloy Test') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" -// load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" -// } -// } -// } -// } -// } -// stage ("Cleanup Dev") { -// steps { -// gitlabCommitStatus(name: 'Cleanup Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// } -// } -// } -// post { -// always { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// success { -// script { -// echo "Milvus Cluster CI/CD success !" -// } -// } -// aborted { -// script { -// echo "Milvus Cluster CI/CD aborted !" -// } -// } -// failure { -// script { -// echo "Milvus Cluster CI/CD failure !" -// } -// } -// } -// } - } - } - } - } - } - - post { - always { - script { - if (env.gitlabAfter != null) { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } - } - } - } - - success { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'success' - echo "Milvus CI/CD success !" - } - } - - aborted { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' - echo "Milvus CI/CD aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'failed' - echo "Milvus CI/CD failure !" - } - } - } -} - diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile deleted file mode 100644 index add9e00f..00000000 --- a/ci/nightly_main_jenkinsfile +++ /dev/null @@ -1,478 +0,0 @@ -pipeline { - agent none - - options { - timestamps() - } - - environment { - PROJECT_NAME = "milvus" - LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() - SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" - GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" - SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" - DOCKER_VERSION_STR = "${env.gitlabAfter == null ? '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, \"yyyyMMdd\"}' : '${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}'}" - } - - stages { - stage("Ubuntu 16.04") { - environment { - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' - ]); - - DOCKER_VERSION = VersionNumber([ - versionNumberString : '${DOCKER_VERSION_STR}' - ]); - } - - stages { - stage("Run Build") { - agent { - kubernetes { - cloud 'build-kubernetes' - label 'build' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - name: milvus-build-env - labels: - app: milvus - componet: build-env -spec: - containers: - - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.13 - command: - - cat - tty: true - resources: - limits: - memory: "28Gi" - cpu: "10.0" - nvidia.com/gpu: 1 - requests: - memory: "14Gi" - cpu: "5.0" -""" - } - } - stages { - stage('Build') { - steps { - gitlabCommitStatus(name: 'Build') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Build', state: 'canceled' - echo "Milvus Build aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Build', state: 'failed' - echo "Milvus Build failure !" - } - } - } - } - - stage("Publish docker and helm") { - agent { - kubernetes { - label 'publish' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-docker - image: registry.zilliz.com/library/zilliz_docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -""" - } - } - stages { - stage('Publish Docker') { - steps { - gitlabCommitStatus(name: 'Publish Docker') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/nightly_publish_docker.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' - echo "Milvus Publish Docker aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' - echo "Milvus Publish Docker failure !" - } - } - } - } - - stage("Deploy to Development") { - parallel { - stage("Single Node") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: test-cluster-config -""" - } - } - - stages { - stage("Deploy to Dev") { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } - } - } - } - } - stage("Dev Test") { - steps { - gitlabCommitStatus(name: 'Deloy Test') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/dev_test_all.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" - } - } - } - } - } - stage ("Cleanup Dev") { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - } - } - } - post { - always { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - success { - script { - echo "Milvus Deploy to Dev Single Node CI/CD success !" - } - } - aborted { - script { - echo "Milvus Deploy to Dev Single Node CI/CD aborted !" - } - } - failure { - script { - echo "Milvus Deploy to Dev Single Node CI/CD failure !" - } - } - } - } - -// stage("Cluster") { -// agent { -// kubernetes { -// label 'dev-test' -// defaultContainer 'jnlp' -// yaml """ -// apiVersion: v1 -// kind: Pod -// metadata: -// labels: -// app: milvus -// componet: test -// spec: -// containers: -// - name: milvus-testframework -// image: registry.zilliz.com/milvus/milvus-test:v0.2 -// command: -// - cat -// tty: true -// volumeMounts: -// - name: kubeconf -// mountPath: /root/.kube/ -// readOnly: true -// volumes: -// - name: kubeconf -// secret: -// secretName: test-cluster-config -// """ -// } -// } -// stages { -// stage("Deploy to Dev") { -// steps { -// gitlabCommitStatus(name: 'Deloy to Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" -// } -// } -// } -// } -// } -// stage("Dev Test") { -// steps { -// gitlabCommitStatus(name: 'Deloy Test') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" -// load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" -// } -// } -// } -// } -// } -// stage ("Cleanup Dev") { -// steps { -// gitlabCommitStatus(name: 'Cleanup Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// } -// } -// } -// post { -// always { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// success { -// script { -// echo "Milvus Deploy to Dev Cluster CI/CD success !" -// } -// } -// aborted { -// script { -// echo "Milvus Deploy to Dev Cluster CI/CD aborted !" -// } -// } -// failure { -// script { -// echo "Milvus Deploy to Dev Cluster CI/CD failure !" -// } -// } -// } -// } - } - } - - stage("Deploy to Staging") { - parallel { - stage("Single Node") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: aks-gpu-cluster-config -""" - } - } - - stages { - stage("Deploy to Staging") { - steps { - gitlabCommitStatus(name: 'Deloy to Staging') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2staging.groovy" - } - } - } - } - } - stage("Staging Test") { - steps { - gitlabCommitStatus(name: 'Staging Test') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/staging_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_staging_test_out.groovy" - } - } - } - } - } - stage ("Cleanup Staging") { - steps { - gitlabCommitStatus(name: 'Cleanup Staging') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" - } - } - } - } - } - } - post { - always { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" - } - } - } - success { - script { - echo "Milvus Deploy to Staging Single Node CI/CD success !" - } - } - aborted { - script { - echo "Milvus Deploy to Staging Single Node CI/CD aborted !" - } - } - failure { - script { - echo "Milvus Deploy to Staging Single Node CI/CD failure !" - } - } - } - } - } - } - } - } - } - - post { - always { - script { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } - } - } - - success { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'success' - echo "Milvus CI/CD success !" - } - } - - aborted { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' - echo "Milvus CI/CD aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'failed' - echo "Milvus CI/CD failure !" - } - } - } -} - diff --git a/ci/pod_containers/milvus-engine-build.yaml b/ci/pod_containers/milvus-engine-build.yaml deleted file mode 100644 index cd5352ff..00000000 --- a/ci/pod_containers/milvus-engine-build.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: build-env -spec: - containers: - - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.9 - command: - - cat - tty: true diff --git a/ci/pod_containers/milvus-testframework.yaml b/ci/pod_containers/milvus-testframework.yaml deleted file mode 100644 index 7a98fbca..00000000 --- a/ci/pod_containers/milvus-testframework.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: testframework -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.1 - command: - - cat - tty: true diff --git a/ci/pod_containers/publish-docker.yaml b/ci/pod_containers/publish-docker.yaml deleted file mode 100644 index 268afb13..00000000 --- a/ci/pod_containers/publish-docker.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-docker - image: registry.zilliz.com/library/zilliz_docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -- GitLab