update Jenkins CI

6535e607 · quicksilver · bf122cc2 · 6535e607 · 6535e607 · 6535e607
12 changed file
--- a/ci/jenkins/Jenkinsfile
+++ b/ci/jenkins/Jenkinsfile
@@ -54,7 +54,7 @@ pipeline {
                }

                stages {
-                    stage ("${OS_NAME} ${CPU_ARCH} ${BINRARY_VERSION}") {
+                    stage("Milvus ${OS_NAME} ${CPU_ARCH} ${BINRARY_VERSION}") {
                        environment {
                            PACKAGE_VERSION = VersionNumber([
                                versionNumberString : '${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
@@ -63,7 +63,7 @@ pipeline {
                        }

                        stages {
-                            stage ("Run Build") {
+                            stage('Run Build') {
                                agent {
                                    kubernetes {
                                        label "${BINRARY_VERSION}-build"
@@ -73,7 +73,7 @@ pipeline {
                                }

                                stages {
-                                    stage ('Build') {
+                                    stage('Build') {
                                        steps {
                                            container("milvus-${BINRARY_VERSION}-build-env") {
                                                script {
@@ -105,7 +105,7 @@ pipeline {
                                }
                            }

-                            stage("Publish docker images") {
+                            stage('Publish docker images') {
                                agent {
                                    kubernetes {
                                        label "${BINRARY_VERSION}-publish"
@@ -127,7 +127,7 @@ pipeline {
                                }
                            }

-                            stage("Deploy to Development") {
+                            stage('Deploy to Development') {
                                environment {
                                    FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-")
                                    HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${BINRARY_VERSION}".toLowerCase()
@@ -142,7 +142,7 @@ pipeline {
                                }

                                stages {
-                                    stage("Deploy to Dev") {
+                                    stage('Deploy to Dev') {
                                        steps {
                                            container('milvus-test-env') {
                                                script {
@@ -152,7 +152,7 @@ pipeline {
                                        }
                                    }

-                                    stage("Dev Test") {
+                                    stage('Dev Test') {
                                        steps {
                                            container('milvus-test-env') {
                                                script {
@@ -167,7 +167,7 @@ pipeline {
                                        }
                                    }

-                                    stage ("Cleanup Dev") {
+                                    stage('Cleanup Dev') {
                                        steps {
                                            container('milvus-test-env') {
                                                script {

--- a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml
@@ -14,8 +14,10 @@ spec:
      valueFrom:
        fieldRef:
          fieldPath: status.podIP
+    - name: OS_NAME
+      value: "centos7"
    - name: BUILD_ENV_IMAGE_ID
-      value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7"
+      value: "225b4d9c26d67b70b476964b4dd6e216de4b464d7a973a8c0c7ed1313c4d81ad"
    command:
    - cat
    tty: true

--- a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml
@@ -14,6 +14,8 @@ spec:
      valueFrom:
        fieldRef:
          fieldPath: status.podIP
+    - name: OS_NAME
+      value: "ubuntu18.04"
    - name: BUILD_ENV_IMAGE_ID
      value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7"
    command:

--- a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml
@@ -14,8 +14,10 @@ spec:
      valueFrom:
        fieldRef:
          fieldPath: status.podIP
+    - name: OS_NAME
+      value: "centos7"
    - name: BUILD_ENV_IMAGE_ID
-      value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea"
+      value: "a5ec9914737ea4727d88ae36b4a73ca5d817f19438ba913cc1de6a1ee2ed2336"
    command:
    - cat
    tty: true

--- a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml
@@ -14,6 +14,8 @@ spec:
      valueFrom:
        fieldRef:
          fieldPath: status.podIP
+    - name: OS_NAME
+      value: "ubuntu18.04"
    - name: BUILD_ENV_IMAGE_ID
      value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea"
    command:

--- a/ci/jenkins/step/build.groovy
+++ b/ci/jenkins/step/build.groovy
@@ -2,7 +2,7 @@ timeout(time: 60, unit: 'MINUTES') {
    dir ("ci/scripts") {
        withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
            def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true)
-            if ("${env.BINRARY_VERSION}" == "gpu") {
+            if ("${BINRARY_VERSION}" == "gpu") {
                sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c"
            } else {
                sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c"

--- a/ci/jenkins/step/deploySingle2Dev.groovy
+++ b/ci/jenkins/step/deploySingle2Dev.groovy
@@ -3,7 +3,7 @@ sh 'helm repo update'
 dir ('milvus-helm') {
    checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]])
    dir ("milvus") {
-        sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
+        sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
    }
 }

--- a/ci/jenkins/step/publishImages.groovy
+++ b/ci/jenkins/step/publishImages.groovy
 timeout(time: 15, unit: 'MINUTES') {
-    dir ("docker/deploy/${env.BINRARY_VERSION}/${env.OS_NAME}") {
+    dir ("docker/deploy/${BINRARY_VERSION}/${OS_NAME}") {
        def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz"

        withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {

--- a/ci/jenkins/step/singleDevNightlyTest.groovy
+++ b/ci/jenkins/step/singleDevNightlyTest.groovy
@@ -13,7 +13,7 @@ timeout(time: 90, unit: 'MINUTES') {
    }
    dir ("milvus-helm") {
        dir ("milvus") {
-            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
+            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
        }
    }
    dir ("tests/milvus_python_test") {

--- a/ci/jenkins/step/singleDevTest.groovy
+++ b/ci/jenkins/step/singleDevTest.groovy
@@ -14,7 +14,7 @@ timeout(time: 60, unit: 'MINUTES') {
    // }
    // dir ("milvus-helm") {
    //     dir ("milvus") {
-    //         sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
+    //         sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
    //     }
    // }
    // dir ("tests/milvus_python_test") {

--- a/ci/scripts/check_ccache.sh
+++ b/ci/scripts/check_ccache.sh
 #!/bin/bash

-OS_NAME="linux"
-CODE_NAME=$(lsb_release -sc)
+OS_NAME="${OS_NAME}"
 BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}"
 BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}')
 ARTIFACTORY_URL=""
@@ -43,13 +42,14 @@ fi

 check_ccache() {
    BRANCH=$1
-    echo "fetching ${BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz"
-    wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz"
+    PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz"
+    echo "fetching ${BRANCH}/${PACKAGE_FILE}"
+    wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}"
    if [[ $? == 0 ]];then
-        wget -q "${ARTIFACTORY_URL}/${BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" && \
+        wget -q "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" && \
        mkdir -p ${CCACHE_DIRECTORY} && \
-        tar zxf ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz -C ${CCACHE_DIRECTORY} && \
-        rm ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz
+        tar zxf ${PACKAGE_FILE} -C ${CCACHE_DIRECTORY} && \
+        rm ${PACKAGE_FILE}
        if [[ $? == 0 ]];then
            echo "found cache"
            exit 0

--- a/ci/scripts/update_ccache.sh
+++ b/ci/scripts/update_ccache.sh
 #!/bin/bash

-OS_NAME="linux"
-CODE_NAME=$(lsb_release -sc)
+OS_NAME="${OS_NAME}"
 BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}"
 BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g')
 ARTIFACTORY_URL=""
@@ -51,7 +50,7 @@ if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then
    exit 1
 fi

-PACKAGE_FILE="ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz"
+PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz"
 REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}"

 ccache --show-stats