From 5bd0de4d81ecbe49d2f9f1e9f80f537f192ffe5b Mon Sep 17 00:00:00 2001 From: Jenny Li Date: Wed, 9 Feb 2022 09:57:46 +0800 Subject: [PATCH] Update CI for QA cluster (#15461) Signed-off-by: Jenny Li --- .../ci/jenkins/{QaPR.groovy => qa/PR.groovy} | 39 ++--- build/ci/jenkins/{pod/qa => qa/pod}/rte.yaml | 13 +- tests/scripts/values/qa/pr.yaml | 161 ++++++++++++++++++ 3 files changed, 190 insertions(+), 23 deletions(-) rename build/ci/jenkins/{QaPR.groovy => qa/PR.groovy} (88%) rename build/ci/jenkins/{pod/qa => qa/pod}/rte.yaml (81%) create mode 100644 tests/scripts/values/qa/pr.yaml diff --git a/build/ci/jenkins/QaPR.groovy b/build/ci/jenkins/qa/PR.groovy similarity index 88% rename from build/ci/jenkins/QaPR.groovy rename to build/ci/jenkins/qa/PR.groovy index 7d6c650d53..60945e8963 100644 --- a/build/ci/jenkins/QaPR.groovy +++ b/build/ci/jenkins/qa/PR.groovy @@ -4,22 +4,21 @@ int total_timeout_minutes = 120 int e2e_timeout_seconds = 70 * 60 def imageTag='' int case_timeout_seconds = 10 * 60 -def chart_version='2.4.25' +def chart_version='3.0.1' + pipeline { options { - timestamps() timeout(time: total_timeout_minutes, unit: 'MINUTES') buildDiscarder logRotator(artifactDaysToKeepStr: '30') parallelsAlwaysFailFast() preserveStashes(buildCount: 5) - } agent { kubernetes { label 'milvus-qa-e2e-test-pipeline' inheritFrom 'default' defaultContainer 'main' - yamlFile 'build/ci/jenkins/pod/qa/rte.yaml' + yamlFile 'build/ci/jenkins/qa/pod/rte.yaml' customWorkspace '/home/jenkins/agent/workspace' } } @@ -28,17 +27,12 @@ pipeline { SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}" DOCKER_BUILDKIT = 1 ARTIFACTS = "${env.WORKSPACE}/_artifacts" - DOCKER_CREDENTIALS_ID = "f0aacc8e-33f2-458a-ba9e-2c44f431b4d2" - TARGET_REPO = "milvusdb" - // CI_DOCKER_CREDENTIAL_ID = "ci-docker-registry" CI_DOCKER_CREDENTIAL_ID = "qa-ci-docker-registry" MILVUS_HELM_NAMESPACE = "milvus-ci" DISABLE_KIND = true - // HUB = 'registry.milvus.io/milvus' HUB = 'harbor.zilliz.cc/milvus-ci' JENKINS_BUILD_ID = "${env.BUILD_ID}" CI_MODE="pr" - MIRROR_URL="http://10.201.20.246:5000" } stages { @@ -108,7 +102,7 @@ pipeline { try{ unstash 'imageTag' imageTag=sh(returnStdout: true, script: 'cat imageTag.txt | tr -d \'\n\r\'') - }catch(e){ + }catch (e){ print "No Image Tag info remained ,please rerun build to build new image." exit 1 } @@ -142,13 +136,13 @@ pipeline { } } } - stage('E2E Test'){ + stage('E2E Test'){ agent { kubernetes { label 'milvus-qa-e2e-test-pr' inheritFrom 'default' defaultContainer 'main' - yamlFile 'build/ci/jenkins/pod/qa/rte.yaml' + yamlFile 'build/ci/jenkins/qa/pod/rte.yaml' customWorkspace '/home/jenkins/agent/workspace' } } @@ -177,23 +171,26 @@ pipeline { } } } - + post{ + always { + container('pytest'){ + dir("${env.ARTIFACTS}") { + sh "tar -zcvf ${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true" + archiveArtifacts artifacts: "${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ", allowEmptyArchive: true + } + } + } + } } } post{ always { - container('pytest'){ - dir("${env.ARTIFACTS}") { - sh "tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true" - archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ", allowEmptyArchive: true - } - } container('main') { dir ('tests/scripts') { script { def release_name=sh(returnStdout: true, script: './get_release_name.sh') - sh "./qa/uninstall_milvus.sh --release-name ${release_name}" - sh "./qa/ci_logs.sh --log-dir /ci-logs --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \ + sh "./uninstall_milvus.sh --release-name ${release_name}" + sh "./ci_logs.sh --log-dir /ci-logs --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \ --release-name ${release_name}" dir("${env.ARTIFACTS}") { archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs.tar.gz", allowEmptyArchive: true diff --git a/build/ci/jenkins/pod/qa/rte.yaml b/build/ci/jenkins/qa/pod/rte.yaml similarity index 81% rename from build/ci/jenkins/pod/qa/rte.yaml rename to build/ci/jenkins/qa/pod/rte.yaml index 031e0fecd5..fbce3e4119 100644 --- a/build/ci/jenkins/pod/qa/rte.yaml +++ b/build/ci/jenkins/qa/pod/rte.yaml @@ -8,7 +8,7 @@ spec: enableServiceLinks: false containers: - name: main - image: harbor.zilliz.cc/dockerhub/milvusdb/krte:20211213-dcc15e9 + image: harbor.zilliz.cc/ci/krte:20211213-dcc15e9 env: - name: DOCKER_IN_DOCKER_ENABLED value: "true" @@ -38,8 +38,10 @@ spec: - mountPath: /mnt/disk/.docker name: build-cache subPath: docker-volume + - mountPath: /ci-logs + name: ci-logs - name: pytest - image: harbor.zilliz.cc/dockerhub/milvusdb/pytest:20220118-3eb5cc4 + image: harbor.zilliz.cc/ci/pytest:20220128-6d9612d resources: limits: cpu: "6" @@ -47,6 +49,9 @@ spec: requests: cpu: "0.5" memory: 5Gi + volumeMounts: + - mountPath: /ci-logs + name: ci-logs volumes: - emptyDir: {} name: docker-graph @@ -64,3 +69,7 @@ spec: path: /sys/fs/cgroup type: Directory name: cgroup + - name: ci-logs + nfs: + path: /volume1/ci-logs + server: 172.16.70.249 diff --git a/tests/scripts/values/qa/pr.yaml b/tests/scripts/values/qa/pr.yaml new file mode 100644 index 0000000000..a130353bd7 --- /dev/null +++ b/tests/scripts/values/qa/pr.yaml @@ -0,0 +1,161 @@ +metrics: + serviceMonitor: + enabled: true + interval: "5s" + scrapeTimeout: "5s" +proxy: + resources: + requests: + cpu: "0.3" + memory: "256Mi" + # limits: + # cpu: "0.5" + # memory: "512Mi" + +rootCoordinator: + resources: + requests: + cpu: "0.3" + memory: "256Mi" + # limits: + # cpu: "0.6" + # memory: "512Mi" + +queryCoordinator: + resources: + requests: + cpu: "0.4" + memory: "100Mi" + # limits: + # cpu: "1.1" + # memory: "256Mi" + +queryNode: + resources: + requests: + cpu: "2" + memory: "500Mi" + limits: + # cpu: "2" + # memory: "1.5Gi" +indexCoordinator: + resources: + requests: + cpu: "0.1" + memory: "50Mi" + # limits: + # cpu: "0.1" + # memory: "100Mi" +indexNode: + resources: + requests: + cpu: "4" + memory: "500Mi" + # limits: + # cpu: "4" + # memory: "1.5Gi" + +dataCoordinator: + resources: + requests: + cpu: "0.1" + memory: "50Mi" + # limits: + # cpu: "0.2" + # memory: "100Mi" + +dataNode: + resources: + requests: + cpu: "0.5" + memory: "500Mi" + # limits: + # cpu: "1.2" + # memory: "1.5Gi" +pulsar: + proxy: + resources: + requests: + cpu: "1" + memory: "2Gi" + # limits: + # cpu: "1" + # memory: "4Gi" + # Resources for the websocket proxy + wsResources: + requests: + memory: "512Mi" + cpu: "0.3" + # limits: + # memory: "512Mi" + # cpu: "0.3" + broker: + resources: + requests: + cpu: "1.5" + memory: "4Gi" + # limits: + # cpu: "1.5" + # memory: "8Gi" + bookkeeper: + volumes: + ledgers: + existingStorageClassName: local-path + resources: + requests: + cpu: 1 + memory: "4Gi" + # limits: + # cpu: 1 + # memory: "8Gi" + bastion: + resources: + requests: + cpu: "0.3" + memory: "50Mi" + # limits: + # cpu: "0.3" + # memory: "50Mi" + autorecovery: + resources: + requests: + cpu: "1" + memory: "512Mi" + # limits: + # cpu: "1" + # memory: "1Gi" + zookeeper: + volumes: + data: + existingStorageClassName: local-path + resources: + requests: + cpu: "0.3" + memory: "1Gi" + # limits: + # cpu: "0.3" + # memory: "2Gi" +etcd: + resources: + requests: + cpu: "0.1" + memory: "100Mi" + # limits: + # cpu: "0.1" + # memory: "300Mi" +minio: + resources: + requests: + cpu: "0.3" + memory: "512Mi" + # limits: + # cpu: "0.6" + # memory: "512Mi" +standalone: + resources: + requests: + cpu: "1" + memory: "3.5Gi" + # limits: + # cpu: "2.6" + # memory: "8.5Gi" \ No newline at end of file