From e1c49cc48bcc55c93c8f7e9ab6db37b47fba816a Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 26 Apr 2020 20:08:07 +0800 Subject: [PATCH] When helm deploy failed, print the milvus container logs (#2130) * When helm deploy failed, print the milvus container logs Signed-off-by: quicksilver * When helm deploy failed, print the milvus container logs Signed-off-by: quicksilver * When helm deploy failed, print the milvus container logs Signed-off-by: quicksilver * When helm deploy failed, print the milvus container logs Signed-off-by: quicksilver * When helm deploy failed, print the milvus container logs Signed-off-by: quicksilver --- ci/jenkins/step/shardsDevNightlyTest.groovy | 12 ++++++++++- ci/jenkins/step/singleDevNightlyTest.groovy | 24 +++++++++++++++++++-- ci/jenkins/step/singleDevTest.groovy | 12 ++++++++++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/ci/jenkins/step/shardsDevNightlyTest.groovy b/ci/jenkins/step/shardsDevNightlyTest.groovy index e152a521..14899752 100644 --- a/ci/jenkins/step/shardsDevNightlyTest.groovy +++ b/ci/jenkins/step/shardsDevNightlyTest.groovy @@ -5,7 +5,17 @@ timeout(time: 180, unit: 'MINUTES') { sh 'helm repo update' checkout([$class: 'GitSCM', branches: [[name: "nightly"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/nightly:refs/remotes/origin/nightly"]]]) retry(3) { - sh "helm install --wait --timeout 300s --set mishards.enabled=true --set persistence.enabled=true --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml --namespace milvus ${env.SHARDS_HELM_RELEASE_NAME} ." + try { + sh "helm install --wait --timeout 300s --set mishards.enabled=true --set persistence.enabled=true --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml --namespace milvus ${env.SHARDS_HELM_RELEASE_NAME} ." + } catch (exc) { + sh "helm get manifest --namespace milvus ${env.SHARDS_HELM_RELEASE_NAME} | kubectl describe -n milvus -f - && \ + kubectl logs --namespace milvus -l \"app=milvus,release=${env.SHARDS_HELM_RELEASE_NAME}\" -c milvus" + def helmResult = sh script: "helm status -n milvus ${env.SHARDS_HELM_RELEASE_NAME}", returnStatus: true + if (!helmResult) { + sh "helm uninstall -n milvus ${env.SHARDS_HELM_RELEASE_NAME}" + } + throw exc + } } } diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index ee0200da..8701e476 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -5,7 +5,17 @@ timeout(time: 180, unit: 'MINUTES') { sh 'helm repo update' checkout([$class: 'GitSCM', branches: [[name: "${env.HELM_BRANCH}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${env.HELM_BRANCH}:refs/remotes/origin/${env.HELM_BRANCH}"]]]) retry(3) { - sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + try { + sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + } catch (exc) { + sh "helm get manifest --namespace milvus ${env.HELM_RELEASE_NAME} | kubectl describe -n milvus -f - && \ + kubectl logs --namespace milvus -l \"app=milvus,release=${env.HELM_RELEASE_NAME}\" -c milvus" + def helmResult = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true + if (!helmResult) { + sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}" + } + throw exc + } } } @@ -24,7 +34,17 @@ timeout(time: 180, unit: 'MINUTES') { } dir ("milvus-helm") { retry(3) { - sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + try { + sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + } catch (exc) { + sh "helm get manifest --namespace milvus ${env.HELM_RELEASE_NAME} | kubectl describe -n milvus -f - && \ + kubectl logs --namespace milvus -l \"app=milvus,release=${env.HELM_RELEASE_NAME}\" -c milvus" + def helmResult = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true + if (!helmResult) { + sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}" + } + throw exc + } } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 21390894..d280e63c 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -5,7 +5,17 @@ timeout(time: 120, unit: 'MINUTES') { sh 'helm repo update' checkout([$class: 'GitSCM', branches: [[name: "${env.HELM_BRANCH}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${env.HELM_BRANCH}:refs/remotes/origin/${env.HELM_BRANCH}"]]]) retry(3) { - sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + try { + sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + } catch (exc) { + sh "helm get manifest --namespace milvus ${env.HELM_RELEASE_NAME} | kubectl describe -n milvus -f - && \ + kubectl logs --namespace milvus -l \"app=milvus,release=${env.HELM_RELEASE_NAME}\" -c milvus" + def helmResult = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true + if (!helmResult) { + sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}" + } + throw exc + } } } dir ("tests/milvus_python_test") { -- GitLab