diff --git a/ci/jenkins/PR.groovy b/ci/jenkins/PR.groovy index f3be503d486de9be8fc025ce70ba0e4dba707aaf..25bc6b13abca8817da4dd9d27af7b0c5aa00ad09 100644 --- a/ci/jenkins/PR.groovy +++ b/ci/jenkins/PR.groovy @@ -17,7 +17,8 @@ pipeline { } agent { kubernetes { - inheritFrom 'milvus-e2e' + cloud '4am' + inheritFrom 'milvus-e2e-4am' defaultContainer 'main' yamlFile 'ci/jenkins/pod/rte-build.yaml' customWorkspace '/home/jenkins/agent/workspace' @@ -42,7 +43,9 @@ pipeline { steps { container('main') { dir ('build'){ - sh './set_docker_mirror.sh' + sh """ + MIRROR_URL="https://docker-nexus-ci.zilliz.cc" ./set_docker_mirror.sh + """ } dir ('tests/scripts') { script { @@ -96,7 +99,7 @@ pipeline { script { sh 'printenv' def clusterEnabled = "false" - def valuesFile = "pr.yaml" + def valuesFile = "pr-4am.yaml" if ("${MILVUS_SERVER_TYPE}".contains('distributed')) { clusterEnabled = "true" } @@ -125,7 +128,7 @@ pipeline { chmod +x /usr/bin/yq ''' sh """ - cp values/ci/pr.yaml values/ci/pr_kafka.yaml + cp values/ci/pr-4am.yaml values/ci/pr_kafka.yaml yq -i '.pulsar.enabled=false' values/ci/pr_kafka.yaml yq -i '.kafka.enabled=true' values/ci/pr_kafka.yaml yq -i '.kafka.metrics.kafka.enabled=true' values/ci/pr_kafka.yaml @@ -136,7 +139,7 @@ pipeline { withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){ sh """ MILVUS_CLUSTER_ENABLED=${clusterEnabled} \ - MILVUS_HELM_REPO="http://nexus-nexus-repository-manager.nexus:8081/repository/milvus-proxy" \ + MILVUS_HELM_REPO="https://nexus-ci.zilliz.cc/repository/milvus-proxy" \ TAG=${imageTag}\ ./e2e-k8s.sh \ --skip-export-logs \ @@ -171,6 +174,7 @@ pipeline { } agent { kubernetes { + cloud '4am' inheritFrom 'default' defaultContainer 'main' yamlFile 'ci/jenkins/pod/e2e.yaml' @@ -196,7 +200,7 @@ pipeline { MILVUS_HELM_NAMESPACE="milvus-ci" \ MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \ TEST_TIMEOUT="${e2e_timeout_seconds}" \ - ./ci_e2e.sh "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}" + ./ci_e2e_4am.sh "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}" """ } else { diff --git a/ci/jenkins/pod/e2e.yaml b/ci/jenkins/pod/e2e.yaml index 01ccd2546ddb147073763bea7960441ab5dbf86e..0999814e9ddc9482b9a4385a1840458a298ce1bd 100644 --- a/ci/jenkins/pod/e2e.yaml +++ b/ci/jenkins/pod/e2e.yaml @@ -23,5 +23,6 @@ spec: volumes: - name: ci-logs nfs: - path: /ci-logs - server: 172.16.70.239 \ No newline at end of file + path: /volume1/ci-logs + # path: /volume1/4am-logs + server: 172.16.70.249 \ No newline at end of file diff --git a/ci/jenkins/pod/rte-build.yaml b/ci/jenkins/pod/rte-build.yaml index a8d6ba5a0b4ad172c729a89b7acbb5c4a9350563..fd39b02006f896547b93a9a48c82b88912ce2235 100644 --- a/ci/jenkins/pod/rte-build.yaml +++ b/ci/jenkins/pod/rte-build.yaml @@ -64,5 +64,6 @@ spec: name: cgroup - name: ci-logs nfs: - path: /ci-logs - server: 172.16.70.239 + path: /volume1/ci-logs + # path: /volume1/4am-logs + server: 172.16.70.249 diff --git a/tests/scripts/ci-util-4am.sh b/tests/scripts/ci-util-4am.sh new file mode 100755 index 0000000000000000000000000000000000000000..03557aec222463e1f012d24d1cc9c4dd32fd9112 --- /dev/null +++ b/tests/scripts/ci-util-4am.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Licensed to the LF AI & Data foundation under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit immediately for non zero status +set -e +# Check unset variables +set -u +# Print commands +set -x + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +ROOT="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )" + + +# Install pytest requirements +function install_pytest_requirements(){ + echo "Install pytest requirements" + cd ${ROOT}/tests/python_client + +# MIRROR_HOST="${MIRROR_HOST:-nexus-nexus-repository-manager.nexus}" + + +export PIP_TRUSTED_HOST="nexus-ci.zilliz.cc" +export PIP_INDEX_URL="https://nexus-ci.zilliz.cc/repository/pypi-all/simple" +export PIP_INDEX="https://nexus-ci.zilliz.cc/repository/pypi-all/pypi" +export PIP_FIND_LINKS="https://nexus-ci.zilliz.cc/repository/pypi-all/pypi" + python3 -m pip install --no-cache-dir -r requirements.txt --timeout 30 --retries 6 +} + +# Login in ci docker registry +function docker_login_ci_registry(){ + + if [[ -z "${CI_REGISTRY_USERNAME:-}" || -z "${CI_REGISTRY_PASSWORD:-}" ]]; then + echo "Please setup docker credential for ci registry-${HUB}" + else + echo "docker login ci registry" + docker login -u ${CI_REGISTRY_USERNAME} -p ${CI_REGISTRY_PASSWORD} ${HUB} + fi +} + diff --git a/tests/scripts/ci_e2e_4am.sh b/tests/scripts/ci_e2e_4am.sh new file mode 100755 index 0000000000000000000000000000000000000000..0640270ab6d518ec8bb35aff74069658224f38ed --- /dev/null +++ b/tests/scripts/ci_e2e_4am.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# Licensed to the LF AI & Data foundation under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +ROOT="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )" + +# Exit immediately for non zero status +set -e +# Check unset variables +set -u +# Print commands +set -x + + + +MILVUS_HELM_RELEASE_NAME="${MILVUS_HELM_RELEASE_NAME:-milvus-testing}" +MILVUS_CLUSTER_ENABLED="${MILVUS_CLUSTER_ENABLED:-false}" +MILVUS_HELM_NAMESPACE="${MILVUS_HELM_NAMESPACE:-default}" +PARALLEL_NUM="${PARALLEL_NUM:-6}" +# Use service name instead of IP to test +MILVUS_SERVICE_NAME=$(echo "${MILVUS_HELM_RELEASE_NAME}-milvus.${MILVUS_HELM_NAMESPACE}" | tr -d '\n') +MILVUS_SERVICE_PORT="19530" +# Minio service name +MINIO_SERVICE_NAME=$(echo "${MILVUS_HELM_RELEASE_NAME}-minio.${MILVUS_HELM_NAMESPACE}" | tr -d '\n') + + +# Shellcheck source=ci-util.sh +source "${ROOT}/tests/scripts/ci-util-4am.sh" + + +cd ${ROOT}/tests/python_client + +# Print python3 version, python version 3.6.8 is more stable for test +python3 -V + +# Pytest will try to get ${CI_LOG_PATH} from environment variables first,then use default path +export CI_LOG_PATH=/tmp/ci_logs/test + +if [ ! -d "${CI_LOG_PATH}" ]; then + # Create dir for ci log path when it does not exist + mkdir -p ${CI_LOG_PATH} +fi + +echo "prepare e2e test" +install_pytest_requirements + + +# Pytest is not able to have both --timeout & --workers, so do not add --timeout or --workers in the shell script +if [[ -n "${TEST_TIMEOUT:-}" ]]; then + + timeout "${TEST_TIMEOUT}" pytest --host ${MILVUS_SERVICE_NAME} --port ${MILVUS_SERVICE_PORT} \ + --html=${CI_LOG_PATH}/report.html --self-contained-html ${@:-} +else + pytest --host ${MILVUS_SERVICE_NAME} --port ${MILVUS_SERVICE_PORT} \ + --html=${CI_LOG_PATH}/report.html --self-contained-html ${@:-} +fi + +# Run bulk insert test +if [[ -n "${TEST_TIMEOUT:-}" ]]; then + + timeout "${TEST_TIMEOUT}" pytest testcases/test_bulk_insert.py --host ${MILVUS_SERVICE_NAME} --port ${MILVUS_SERVICE_PORT} --minio_host ${MINIO_SERVICE_NAME} \ + --html=${CI_LOG_PATH}/report_bulk_insert.html --self-contained-html +else + pytest testcases/test_bulk_insert.py --host ${MILVUS_SERVICE_NAME} --port ${MILVUS_SERVICE_PORT} --minio_host ${MINIO_SERVICE_NAME} \ + --html=${CI_LOG_PATH}/report_bulk_insert.html --self-contained-html +fi \ No newline at end of file diff --git a/tests/scripts/ci_logs.sh b/tests/scripts/ci_logs.sh index 0a6732616f6cc2219328de5a136a028465968b30..a2e2a728a70324e817348156d5350115434669a2 100755 --- a/tests/scripts/ci_logs.sh +++ b/tests/scripts/ci_logs.sh @@ -87,14 +87,19 @@ if [[ ! -d ${RELEASE_LOG_DIR} ]] ;then fi # Try to found logs file from mount disk /volume1/ci-logs log_files=$(find ${LOG_DIR} -type f -name "*${RELEASE_NAME}*" ) -for log_file in ${log_files} -do - file_name=$(basename ${log_file}) - mv ${log_file} ${RELEASE_LOG_DIR}/`echo ${file_name} | sed 's/jenkins.var.log.containers.//g' ` -done - -tar -zcvf ${ARTIFACTS_NAME:-artifacts}.tar.gz ${RELEASE_LOG_DIR}/* -rm -rf ${RELEASE_LOG_DIR} + +if [ -z "${log_files:-}" ]; then + echo "No log files find" +else + for log_file in ${log_files} + do + file_name=$(basename ${log_file}) + mv ${log_file} ${RELEASE_LOG_DIR}/`echo ${file_name} | sed 's/jenkins.var.log.containers.//g' ` + done + + tar -zcvf ${ARTIFACTS_NAME:-artifacts}.tar.gz ${RELEASE_LOG_DIR}/* + rm -rf ${RELEASE_LOG_DIR} +fi remain_log_files=$(find ${LOG_DIR} -type f -name "*${RELEASE_NAME}*") diff --git a/tests/scripts/values/ci/pr-4am.yaml b/tests/scripts/values/ci/pr-4am.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b2cacd7a0296864a6374aa938abdf9abbbdcd77 --- /dev/null +++ b/tests/scripts/values/ci/pr-4am.yaml @@ -0,0 +1,186 @@ +metrics: + serviceMonitor: + enabled: true +log: + level: debug +proxy: + resources: + requests: + cpu: "0.3" + memory: "256Mi" + limits: + cpu: "1" +rootCoordinator: + resources: + requests: + cpu: "0.2" + memory: "256Mi" + limits: + cpu: "1" +queryCoordinator: + resources: + requests: + cpu: "0.2" + memory: "100Mi" + limits: + cpu: "1" +queryNode: + resources: + requests: + cpu: "0.5" + memory: "500Mi" + limits: + cpu: "2" +indexCoordinator: + resources: + requests: + cpu: "0.1" + memory: "50Mi" + limits: + cpu: "1" +indexNode: + resources: + requests: + cpu: "0.5" + memory: "500Mi" + limits: + cpu: "2" +dataCoordinator: + resources: + requests: + cpu: "0.1" + memory: "50Mi" + limits: + cpu: "1" +dataNode: + resources: + requests: + cpu: "0.5" + memory: "500Mi" + limits: + cpu: "2" + +pulsar: + components: + autorecovery: false + proxy: + configData: + PULSAR_MEM: > + -Xms1024m -Xmx1024m + PULSAR_GC: > + -XX:MaxDirectMemorySize=2048m + httpNumThreads: "50" + resources: + requests: + cpu: "0.5" + memory: "1Gi" + # Resources for the websocket proxy + wsResources: + requests: + memory: "100Mi" + cpu: "0.1" + broker: + resources: + requests: + cpu: "0.5" + memory: "4Gi" + configData: + PULSAR_MEM: > + -Xms4096m + -Xmx4096m + -XX:MaxDirectMemorySize=8192m + PULSAR_GC: > + -Dio.netty.leakDetectionLevel=disabled + -Dio.netty.recycler.linkCapacity=1024 + -XX:+ParallelRefProcEnabled + -XX:+UnlockExperimentalVMOptions + -XX:+DoEscapeAnalysis + -XX:ParallelGCThreads=32 + -XX:ConcGCThreads=32 + -XX:G1NewSizePercent=50 + -XX:+DisableExplicitGC + -XX:-ResizePLAB + -XX:+ExitOnOutOfMemoryError + maxMessageSize: "104857600" + defaultRetentionTimeInMinutes: "10080" + defaultRetentionSizeInMB: "8192" + backlogQuotaDefaultLimitGB: "8" + backlogQuotaDefaultRetentionPolicy: producer_exception + + bookkeeper: + configData: + PULSAR_MEM: > + -Xms4096m + -Xmx4096m + -XX:MaxDirectMemorySize=8192m + PULSAR_GC: > + -Dio.netty.leakDetectionLevel=disabled + -Dio.netty.recycler.linkCapacity=1024 + -XX:+UseG1GC -XX:MaxGCPauseMillis=10 + -XX:+ParallelRefProcEnabled + -XX:+UnlockExperimentalVMOptions + -XX:+DoEscapeAnalysis + -XX:ParallelGCThreads=32 + -XX:ConcGCThreads=32 + -XX:G1NewSizePercent=50 + -XX:+DisableExplicitGC + -XX:-ResizePLAB + -XX:+ExitOnOutOfMemoryError + -XX:+PerfDisableSharedMem + -XX:+PrintGCDetails + nettyMaxFrameSizeBytes: "104867840" + resources: + requests: + cpu: "0.5" + memory: "4Gi" + + zookeeper: + replicaCount: 1 + configData: + PULSAR_MEM: > + -Xms1024m + -Xmx1024m + PULSAR_GC: > + -Dcom.sun.management.jmxremote + -Djute.maxbuffer=10485760 + -XX:+ParallelRefProcEnabled + -XX:+UnlockExperimentalVMOptions + -XX:+DoEscapeAnalysis + -XX:+DisableExplicitGC + -XX:+PerfDisableSharedMem + -Dzookeeper.forceSync=no + resources: + requests: + cpu: "0.3" + memory: "512Mi" +kafka: + resources: + requests: + cpu: "0.5" + memory: "1Gi" + zookeeper: + replicaCount: 1 + resources: + requests: + cpu: "0.3" + memory: "512Mi" +etcd: + nodeSelector: + nvme: "true" + replicaCount: 1 + resources: + requests: + cpu: "0.3" + memory: "100Mi" +minio: + resources: + requests: + cpu: "0.3" + memory: "512Mi" +standalone: + resources: + requests: + cpu: "1" + memory: "3.5Gi" + limits: + cpu: "4"