diff --git a/.jenkins/modules/Regression/PythonRegression.groovy b/.jenkins/modules/Regression/PythonRegression.groovy index 9e64880798599d0a72d243a03aae0e727c123262..42528e69e3bd341cdd94176fb2532240b44be8e9 100644 --- a/.jenkins/modules/Regression/PythonRegression.groovy +++ b/.jenkins/modules/Regression/PythonRegression.groovy @@ -1,30 +1,30 @@ try { - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d etcd' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d pulsar' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d minio' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d etcd' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d pulsar' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d minio' dir ('build/docker/deploy') { - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} pull' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d master' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d indexservice' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d indexnode' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d proxyservice' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d dataservice' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d queryservice' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run -e DATA_NODE_ID=3 -d datanode' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d proxynode' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run -e QUERY_NODE_ID=1 -d querynode' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run -e QUERY_NODE_ID=2 -d querynode' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} pull' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d master' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d indexservice' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d indexnode' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d proxyservice' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d dataservice' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d queryservice' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e DATA_NODE_ID=3 -d datanode' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d proxynode' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e QUERY_NODE_ID=1 -d querynode' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e QUERY_NODE_ID=2 -d querynode' } dir ('build/docker/test') { sh 'docker pull ${SOURCE_REPO}/pytest:${SOURCE_TAG} || true' - sh 'docker-compose build --force-rm regression' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run --rm regression' + sh 'docker-compose build --force-rm ${REGRESSION_SERVICE_NAME}' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run --rm ${REGRESSION_SERVICE_NAME}' try { withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) { sh 'docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${DOKCER_REGISTRY_URL}' - sh 'docker-compose push regression || true' + sh 'docker-compose push ${REGRESSION_SERVICE_NAME} || true' } } catch (exc) { throw exc @@ -36,15 +36,15 @@ try { throw exc } finally { dir ('build/docker/deploy') { - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} ps -a | tail -n +3 | awk \'{ print $1 }\' | ( while read arg; do docker logs -t $arg > $arg.log 2>&1; done )' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} ps -a | tail -n +3 | awk \'{ print $1 }\' | ( while read arg; do docker logs -t $arg > $arg.log 2>&1; done )' archiveArtifacts artifacts: "**.log", allowEmptyArchive: true - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} down --rmi all -v || true' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} down --rmi all -v || true' } - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} rm -f -s -v pulsar' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} rm -f -s -v etcd' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} rm -f -s -v minio' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v pulsar' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v etcd' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v minio' dir ('build/docker/test') { - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run --rm regression /bin/bash -c "rm -rf __pycache__ && rm -rf .pytest_cache"' - sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} down --rmi all -v || true' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run --rm ${REGRESSION_SERVICE_NAME} /bin/bash -c "rm -rf __pycache__ && rm -rf .pytest_cache"' + sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} down --rmi all -v || true' } } diff --git a/build/ci/jenkins/Jenkinsfile b/build/ci/jenkins/Jenkinsfile index 570e1a9d07467394d0669833ea65583c1647a8d7..364d8bccdbd53e9164300ad830ac95faeb9e3032 100644 --- a/build/ci/jenkins/Jenkinsfile +++ b/build/ci/jenkins/Jenkinsfile @@ -68,18 +68,30 @@ pipeline { } stage ('Dev Test') { - agent { - label "performance" - } - environment { - DOCKER_COMPOSE_PROJECT_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}".replaceAll("\\.", "-").replaceAll("_", "-") - } - steps { - MPLModule('Python Regression') - } - post { - cleanup { - deleteDir() /* clean up our workspace */ + matrix { + axes { + axis { + name 'REGRESSION_SERVICE_NAME' + values 'regression' + } + } + environment { + DOCKER_COMPOSE_PROJECT_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}".replaceAll("\\.", "-").replaceAll("_", "-") + } + agent { + label 'performance' + } + stages { + stage('Test') { + steps { + MPLModule('Python Regression') + } + } + } + post { + cleanup { + deleteDir() /* clean up our workspace */ + } } } } diff --git a/build/docker/deploy/distributed/Dockerfile b/build/docker/deploy/distributed/Dockerfile index ce1167e3830e4e601947a637885d91591006949f..9eac4d622f60fcab98e93934d9e40a587715673a 100644 --- a/build/docker/deploy/distributed/Dockerfile +++ b/build/docker/deploy/distributed/Dockerfile @@ -11,6 +11,8 @@ FROM milvusdb/milvus-distributed-dev:amd64-ubuntu18.04-latest AS openblas +FROM hectormolinero/tini:v18 AS tini + #FROM alpine FROM ubuntu:bionic-20200921 @@ -19,11 +21,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends libtbb-dev gfor rm -rf /var/lib/apt/lists/* # Add Tini -ENV TINI_VERSION v0.19.0 - -ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini - -RUN chmod +x /tini +COPY --from=tini /usr/bin/tini /tini COPY --from=openblas /usr/lib/libopenblas-r0.3.9.so /usr/lib/ diff --git a/build/docker/test/Dockerfile b/build/docker/test/Dockerfile index 36bdeadf540713523ba0acc22987754a5a28a7e9..a5833545669edd8e4eb7d6db162d4c7a566f36db 100644 --- a/build/docker/test/Dockerfile +++ b/build/docker/test/Dockerfile @@ -13,6 +13,6 @@ FROM python:3.6.8-jessie COPY ./tests/python/requirements.txt /requirements.txt -RUN python3 -m pip install -r /requirements.txt +RUN python3 -m pip install -r /requirements.txt && python3 -m pip install git+https://gitee.com/quicksilver/pytest-tags.git CMD ["tail", "-f", "/dev/null"] diff --git a/build/docker/test/docker-compose.yml b/build/docker/test/docker-compose.yml index caf095f411a32bda3fef3d01c85a85300a760343..6bff0344dc605a9b4008a6da644bb9513d2e1fa7 100644 --- a/build/docker/test/docker-compose.yml +++ b/build/docker/test/docker-compose.yml @@ -16,5 +16,20 @@ services: networks: - milvus + regression_0331: + image: ${TARGET_REPO}/pytest:${TARGET_TAG} + build: + context: ../../../ + dockerfile: build/docker/test/Dockerfile + cache_from: + - ${SOURCE_REPO}/pytest:${SOURCE_TAG} + volumes: + - ../../..:/milvus-distributed:delegated + working_dir: "/milvus-distributed/tests/python_test" + command: > + /bin/bash -c "pytest --tags=0331 --ip proxyservice -n 4" + networks: + - milvus + networks: milvus: