diff --git a/.jenkins/modules/Regression/CleanupPythonRegression.groovy b/.jenkins/modules/Regression/CleanupPythonRegression.groovy new file mode 100644 index 0000000000000000000000000000000000000000..d454c99664701400998550d842036d8232d6b833 --- /dev/null +++ b/.jenkins/modules/Regression/CleanupPythonRegression.groovy @@ -0,0 +1,8 @@ +container('deploy-env') { + def helmStatus = sh script: "helm status -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}", returnStatus: true + if (!helmStatus) { + sh "helm uninstall -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}" + def labels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME}" + sh "kubectl delete pvc -n ${env.HELM_RELEASE_NAMESPACE} \$(kubectl get pvc -n ${env.HELM_RELEASE_NAMESPACE} -l ${labels} -o jsonpath='{range.items[*]}{.metadata.name} ')" + } +} diff --git a/.jenkins/modules/Regression/PythonRegression.groovy b/.jenkins/modules/Regression/PythonRegression.groovy index 0d680d469ce9ef71c8f1fadf95f3fbf26c9d473a..19443b67d6d516d0112a5f3f11e12f9e7e6a2b8c 100644 --- a/.jenkins/modules/Regression/PythonRegression.groovy +++ b/.jenkins/modules/Regression/PythonRegression.groovy @@ -1,59 +1,70 @@ timeout(time: 60, unit: 'MINUTES') { - try { - if ("${REGRESSION_SERVICE_NAME}" == "regression_distributed") { - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d pulsar' - } - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d etcd' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d minio' - dir ('build/docker/deploy') { - sh 'docker pull ${TARGET_REPO}/milvus-distributed:${TARGET_TAG}' - if ("${REGRESSION_SERVICE_NAME}" == "regression_distributed") { - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d master' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d indexservice' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d indexnode' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d proxyservice' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d dataservice' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d queryservice' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e DATA_NODE_ID=3 -d datanode' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d proxynode' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e QUERY_NODE_ID=1 -d querynode' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e QUERY_NODE_ID=2 -d querynode' - } else { - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d standalone' + container('deploy-env') { + dir ('milvus-helm-chart') { + sh " helm version && \ + helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts && \ + helm repo add bitnami https://charts.bitnami.com/bitnami && \ + helm repo add minio https://helm.min.io/ && \ + helm repo update" + + def milvusHelmURL = "https://github.com/zilliztech/milvus-helm-charts.git" + checkout([$class: 'GitSCM', branches: [[name: "${env.HELM_BRANCH}"]], userRemoteConfigs: [[url: "${milvusHelmURL}"]]]) + + dir ('charts/milvus-ha') { + sh script: "kubectl create namespace ${env.HELM_RELEASE_NAMESPACE}", returnStatus: true + + def helmCMD = "" + if ("${REGRESSION_SERVICE_TYPE}" == "distributed") { + helmCMD = "helm install --wait --timeout 300s \ + --set standalone.enabled=false \ + --set image.all.repository=${env.TARGET_REPO}/milvus-distributed \ + --set image.all.tag=${env.TARGET_TAG} \ + --set image.all.pullPolicy=Always \ + --set logsPersistence.enabled=true \ + --set logsPersistence.mountPath=/milvus-distributed/logs \ + --namespace ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} ." + } else { + helmCMD = "helm install --wait --timeout 300s \ + --set image.all.repository=${env.TARGET_REPO}/milvus-distributed \ + --set image.all.tag=${env.TARGET_TAG} \ + --set image.all.pullPolicy=Always \ + --set logsPersistence.enabled=true \ + --set logsPersistence.mountPath=/milvus-distributed/logs \ + --namespace ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} ." + } + + try { + sh "${helmCMD}" + } catch (exc) { + def helmStatusCMD = "helm get manifest -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} | kubectl describe -n ${env.HELM_RELEASE_NAMESPACE} -f - && \ + helm status -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}" + sh script: helmStatusCMD, returnStatus: true + throw exc + } } } + } - dir ('build/docker/test') { - sh 'docker pull ${SOURCE_REPO}/pytest:${SOURCE_TAG} || true' - sh 'docker-compose build --force-rm ${REGRESSION_SERVICE_NAME}' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run --rm ${REGRESSION_SERVICE_NAME}' - try { - withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) { - sh 'docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${DOKCER_REGISTRY_URL}' - sh 'docker-compose push ${REGRESSION_SERVICE_NAME} || true' + container('test-env') { + try { + dir ('tests/python_test') { + sh "pytest --tags=0331 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local" + } + } catch (exc) { + echo 'PyTest Regression Failed !' + throw exc + } finally { + container('deploy-env') { + def labels = "" + if ("${REGRESSION_SERVICE_TYPE}" == "distributed") { + labels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},component=proxyservice" + } else { + labels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},component=standalone" } - } catch (exc) { - throw exc - } finally { - sh 'docker logout ${DOKCER_REGISTRY_URL}' + sh "mkdir -p ${env.DEV_TEST_ARTIFACTS_PATH}" + sh "kubectl cp -n ${env.HELM_RELEASE_NAMESPACE} \$(kubectl get pod -n ${env.HELM_RELEASE_NAMESPACE} -l ${labels} -o jsonpath='{range.items[0]}{.metadata.name}'):logs ${env.DEV_TEST_ARTIFACTS_PATH}" + archiveArtifacts artifacts: "${env.DEV_TEST_ARTIFACTS_PATH}/**", allowEmptyArchive: true } } - } catch(exc) { - throw exc - } finally { - dir ('build/docker/deploy') { - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} ps -a | tail -n +3 | awk \'{ print $1 }\' | ( while read arg; do docker logs -t $arg > $arg.log 2>&1; done )' - archiveArtifacts artifacts: "**.log", allowEmptyArchive: true - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} down --rmi all -v || true' - } - if ("${REGRESSION_SERVICE_NAME}" == "regression_distributed") { - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v pulsar' - } - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v etcd' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v minio' - dir ('build/docker/test') { - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run --rm ${REGRESSION_SERVICE_NAME} /bin/bash -c "find . -name \'__pycache__\' | xargs rm -rf && find . -name \'.pytest_cache\' | xargs rm -rf"' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} down --rmi all -v || true' - } } } diff --git a/build/ci/jenkins/Jenkinsfile b/build/ci/jenkins/Jenkinsfile index e1ea9e53efb5445796337e0ef473e209dd1d4a2a..5ff3ec25551da90793ff7a040d7dc2130c59b67e 100644 --- a/build/ci/jenkins/Jenkinsfile +++ b/build/ci/jenkins/Jenkinsfile @@ -7,7 +7,7 @@ pipeline { timestamps() } environment { - PROJECT_NAME = "milvus-distributed" + PROJECT_NAME = "milvus" SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}" BUILD_TYPE = "Release" LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() @@ -71,15 +71,22 @@ pipeline { matrix { axes { axis { - name 'REGRESSION_SERVICE_NAME' - values 'regression_standalone', 'regression_distributed' + name 'REGRESSION_SERVICE_TYPE' + values 'standalone', 'distributed' } } environment { - DOCKER_COMPOSE_PROJECT_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}".replaceAll("\\.", "-").replaceAll("_", "-") + HELM_BRANCH = "main" + HELM_RELEASE_NAMESPACE = "dev" + HELM_RELEASE_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-${REGRESSION_SERVICE_TYPE}".replaceAll("\\.", "-").replaceAll("_", "-").toLowerCase() + DEV_TEST_ARTIFACTS_PATH = "artifacts" } agent { - label 'performance' + kubernetes { + label "${env.PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-${REGRESSION_SERVICE_TYPE}-devtest" + defaultContainer 'test-env' + yamlFile "build/ci/jenkins/pod/test-env.yaml" + } } stages { stage('Test') { @@ -90,7 +97,7 @@ pipeline { } post { cleanup { - deleteDir() /* clean up our workspace */ + MPLModule('Cleanup Python Regression') } } } diff --git a/build/ci/jenkins/pod/test-env.yaml b/build/ci/jenkins/pod/test-env.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f92c718c861e031ad42ce48b108d17a57144a019 --- /dev/null +++ b/build/ci/jenkins/pod/test-env.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test-env +spec: + containers: + - name: test-env + image: milvusdb/pytest:20210310-5ea22d7 + command: + - cat + tty: true + resources: + limits: + memory: "8Gi" + cpu: "4.0" + requests: + memory: "4Gi" + cpu: "2.0" + - name: deploy-env + image: registry.zilliz.com/library/helm3:v0.1 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config + tolerations: + - key: dedicated + operator: Equal + value: milvus + effect: NoSchedule diff --git a/build/docker/test/.env b/build/docker/test/.env index f94e4c5d5d3e17ef79abbf626a1c0638c4adf291..fbebcd2ce861b7bafb6f8516bbd0cc7657a6b4c1 100644 --- a/build/docker/test/.env +++ b/build/docker/test/.env @@ -1,4 +1,4 @@ SOURCE_REPO=milvusdb TARGET_REPO=milvusdb -SOURCE_TAG=latest -TARGET_TAG=latest \ No newline at end of file +SOURCE_TAG=20210310-5ea22d7 +TARGET_TAG=20210310-5ea22d7 \ No newline at end of file