diff --git a/.github/workflows/jenkins-checker.yaml b/.github/workflows/jenkins-checker.yaml index 72080f7709..29a9d1b802 100644 --- a/.github/workflows/jenkins-checker.yaml +++ b/.github/workflows/jenkins-checker.yaml @@ -7,6 +7,7 @@ on: # file paths to consider in the event. Optional; defaults to all. paths: - 'build/ci/jenkins/**.groovy' + - 'ci/jenkins/**.groovy' - '.github/workflows/jenkins-checker.yaml' jobs: check-jenkinsfile: @@ -18,13 +19,12 @@ jobs: - name: Validate Jenkinsfile shell: bash run: | - JENKINS_URL=https://ci.milvus.io:18080/jenkins - # JENKINS_CRUMB is needed if your Jenkins controller has CRSF protection enabled as it should - JENKINS_CRUMB=`curl "$JENKINS_URL/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,\":\",//crumb)"` - function validate(){ local file_path=${1:-Jenkinsfile} - response=$(curl --max-time 10 --retry 5 --retry-delay 0 --retry-max-time 40 -X POST -H $JENKINS_CRUMB -F "jenkinsfile=<${file_path}" $JENKINS_URL/pipeline-model-converter/validate) + local jenkins_url=${2:-"https://ci.milvus.io:18080/jenkins"} + + JENKINS_CRUMB=`curl "${jenkins_url}/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,\":\",//crumb)"` + response=$(curl --max-time 10 --retry 5 --retry-delay 0 --retry-max-time 40 -X POST -H $JENKINS_CRUMB -F "jenkinsfile=<${file_path}" ${jenkins_url}/pipeline-model-converter/validate) if [[ ${response} =~ "Error" ]] then @@ -34,24 +34,28 @@ jobs: exit 1 fi } + function validate_path(){ + local path=${1} + local jenkins_url=${2} - for file in build/ci/jenkins/* - do - if [ -f "$file" ] - then - echo "$file" - file_name=$(basename "$file") - - if echo "${file_name}" | grep -q -E '\.groovy$' - then - echo "Validate groovy file ${file_name}" - validate $file - elif [[ "${file_name}" == "Jenkinsfile" ]] - then - echo "Validate Jenkinsfile" - validate $file - fi - fi - done - + for file in ${path} + do + if [ -f "$file" ] + then + # echo "$file" + file_name=$(basename "$file") + if echo "${file_name}" | grep -q -E '\.groovy$' + then + # echo "Validate groovy file ${file_name}" + validate $file ${jenkins_url} + elif [[ "${file_name}" == "Jenkinsfile" ]] + then + # echo "Validate Jenkinsfile" + validate $file ${jenkins_url} + fi + fi + done + } + validate_path "build/ci/jenkins/*" "https://ci.milvus.io:18080/jenkins/" + validate_path "ci/jenkins/*" "https://jenkins.milvus.io:18080/" \ No newline at end of file diff --git a/ci/jenkins/PR.groovy b/ci/jenkins/PR.groovy new file mode 100644 index 0000000000..7888bff38e --- /dev/null +++ b/ci/jenkins/PR.groovy @@ -0,0 +1,232 @@ +#!/usr/bin/env groovy + +int total_timeout_minutes = 120 +int e2e_timeout_seconds = 70 * 60 +def imageTag='' +int case_timeout_seconds = 10 * 60 +def chart_version='3.0.27' +pipeline { + options { + timeout(time: total_timeout_minutes, unit: 'MINUTES') + buildDiscarder logRotator(artifactDaysToKeepStr: '30') + // parallelsAlwaysFailFast() + preserveStashes(buildCount: 5) + disableConcurrentBuilds(abortPrevious: true) + + } + agent { + kubernetes { + label 'milvus-e2e-test-pipeline' + inheritFrom 'default' + defaultContainer 'main' + yamlFile 'ci/jenkins/pod/rte.yaml' + customWorkspace '/home/jenkins/agent/workspace' + } + } + environment { + PROJECT_NAME = 'milvus' + SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}" + DOCKER_BUILDKIT = 1 + ARTIFACTS = "${env.WORKSPACE}/_artifacts" + CI_DOCKER_CREDENTIAL_ID = "harbor-milvus-io-registry" + MILVUS_HELM_NAMESPACE = "milvus-ci" + DISABLE_KIND = true + HUB = 'harbor.milvus.io/milvus' + JENKINS_BUILD_ID = "${env.BUILD_ID}" + CI_MODE="pr" + } + + stages { + stage ('Build'){ + steps { + container('main') { + dir ('build'){ + sh './set_docker_mirror.sh' + } + dir ('tests/scripts') { + script { + sh 'printenv' + def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim() + def gitShortCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim() + imageTag="${env.BRANCH_NAME}-${date}-${gitShortCommit}" + withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){ + sh """ + TAG="${imageTag}" \ + ./e2e-k8s.sh \ + --skip-export-logs \ + --skip-install \ + --skip-cleanup \ + --skip-setup \ + --skip-test + """ + + // stash imageTag info for rebuild install & E2E Test only + sh "echo ${imageTag} > imageTag.txt" + stash includes: 'imageTag.txt', name: 'imageTag' + + } + } + } + } + } + } + + + stage('Install & E2E Test') { + matrix { + axes { + axis { + name 'MILVUS_SERVER_TYPE' + values 'standalone', 'distributed' + } + axis { + name 'MILVUS_CLIENT' + values 'pymilvus' + } + } + + stages { + stage('Install') { + steps { + container('main') { + stash includes: 'tests/**', name: 'testCode', useDefaultExcludes: false + dir ('tests/scripts') { + script { + sh 'printenv' + def clusterEnabled = "false" + if ("${MILVUS_SERVER_TYPE}" == 'distributed') { + clusterEnabled = "true" + } + + if ("${MILVUS_CLIENT}" == "pymilvus") { + if ("${imageTag}"==''){ + dir ("imageTag"){ + try{ + unstash 'imageTag' + imageTag=sh(returnStdout: true, script: 'cat imageTag.txt | tr -d \'\n\r\'') + }catch(e){ + print "No Image Tag info remained ,please rerun build to build new image." + exit 1 + } + } + } + withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){ + sh """ + MILVUS_CLUSTER_ENABLED=${clusterEnabled} \ + TAG=${imageTag}\ + ./e2e-k8s.sh \ + --skip-export-logs \ + --skip-cleanup \ + --skip-setup \ + --skip-test \ + --skip-build \ + --skip-build-image \ + --install-extra-arg " + --set etcd.metrics.enabled=true \ + --set etcd.metrics.podMonitor.enabled=true \ + --set indexCoordinator.gc.interval=1 \ + --version ${chart_version} \ + -f values/ci/pr.yaml" + """ + } + } else { + error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}" + } + } + } + } + + } + } + stage('E2E Test'){ + options { + skipDefaultCheckout() + } + agent { + kubernetes { + label 'milvus-e2e-test-pr' + inheritFrom 'default' + defaultContainer 'main' + yamlFile 'ci/jenkins/pod/rte.yaml' + customWorkspace '/home/jenkins/agent/workspace' + } + } + steps { + container('pytest') { + unstash('testCode') + script { + sh 'ls -lah' + } + dir ('tests/scripts') { + script { + def release_name=sh(returnStdout: true, script: './get_release_name.sh') + def clusterEnabled = 'false' + if ("${MILVUS_SERVER_TYPE}" == "distributed") { + clusterEnabled = "true" + } + if ("${MILVUS_CLIENT}" == "pymilvus") { + sh """ + MILVUS_HELM_RELEASE_NAME="${release_name}" \ + MILVUS_HELM_NAMESPACE="milvus-ci" \ + MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \ + TEST_TIMEOUT="${e2e_timeout_seconds}" \ + ./ci_e2e.sh "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}" + """ + + } else { + error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}" + } + } + } + } + } + post{ + always { + container('pytest'){ + dir("${env.ARTIFACTS}") { + sh "tar -zcvf ${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true" + archiveArtifacts artifacts: "${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ", allowEmptyArchive: true + } + } + } + } + } + } + post{ + always { + container('main') { + dir ('tests/scripts') { + script { + def release_name=sh(returnStdout: true, script: './get_release_name.sh') + sh "./uninstall_milvus.sh --release-name ${release_name}" + sh "./ci_logs.sh --log-dir /ci-logs --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \ + --release-name ${release_name}" + dir("${env.ARTIFACTS}") { + archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs.tar.gz", allowEmptyArchive: true + } + } + } + } + } + } + } + + } + } + post{ + unsuccessful { + container('jnlp') { + dir ('tests/scripts') { + script { + def authorEmail = sh(returnStdout: true, script: './get_author_email.sh ') + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [developers(), culprits()], + replyTo: '$DEFAULT_REPLYTO', + to: "${authorEmail},devops@zilliz.com" + } + } + } + } + } +} diff --git a/ci/jenkins/pod/rte.yaml b/ci/jenkins/pod/rte.yaml new file mode 100644 index 0000000000..c3436a8f76 --- /dev/null +++ b/ci/jenkins/pod/rte.yaml @@ -0,0 +1,75 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus-e2e + namespace: jenkins +spec: + enableServiceLinks: false + containers: + - name: main + image: milvusdb/krte:20211213-dcc15e9 + env: + - name: DOCKER_IN_DOCKER_ENABLED + value: "true" + - name: DOCKER_VOLUME_DIRECTORY + value: "/mnt/disk/.docker" + tty: true + securityContext: + privileged: true + args: ["cat"] + resources: + limits: + cpu: "6" + memory: 12Gi + requests: + cpu: "0.5" + memory: 5Gi + volumeMounts: + - mountPath: /docker-graph + name: docker-graph + - mountPath: /var/lib/docker + name: docker-root + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /sys/fs/cgroup + name: cgroup + - mountPath: /mnt/disk/.docker + name: build-cache + subPath: docker-volume + - mountPath: /ci-logs + name: ci-logs + - name: pytest + image: milvusdb/pytest:20220705-e979e73 + resources: + limits: + cpu: "6" + memory: 12Gi + requests: + cpu: "0.5" + memory: 5Gi + volumeMounts: + - mountPath: /ci-logs + name: ci-logs + volumes: + - emptyDir: {} + name: docker-graph + - emptyDir: {} + name: docker-root + - hostPath: + path: /tmp/krte/cache + type: DirectoryOrCreate + name: build-cache + - hostPath: + path: /lib/modules + type: Directory + name: modules + - hostPath: + path: /sys/fs/cgroup + type: Directory + name: cgroup + - name: ci-logs + nfs: + path: /ci-logs + server: 172.16.70.239 diff --git a/tests/scripts/ci-util.sh b/tests/scripts/ci-util.sh index 8bbe1c3b6e..079d4bb753 100755 --- a/tests/scripts/ci-util.sh +++ b/tests/scripts/ci-util.sh @@ -41,7 +41,6 @@ export PIP_TRUSTED_HOST="nexus-nexus-repository-manager.nexus" export PIP_INDEX_URL="http://nexus-nexus-repository-manager.nexus:8081/repository/pypi-all/simple" export PIP_INDEX="http://nexus-nexus-repository-manager.nexus:8081/repository/pypi-all/pypi" export PIP_FIND_LINKS="http://nexus-nexus-repository-manager.nexus:8081/repository/pypi-all/pypi" - python3 -m pip install --upgrade setuptools python3 -m pip install --no-cache-dir -r requirements.txt --timeout 30 --retries 6 } diff --git a/tests/scripts/values/ci/pr.yaml b/tests/scripts/values/ci/pr.yaml new file mode 100644 index 0000000000..05772fb1f7 --- /dev/null +++ b/tests/scripts/values/ci/pr.yaml @@ -0,0 +1,166 @@ +metrics: + serviceMonitor: + enabled: true +proxy: + resources: + requests: + cpu: "0.3" + memory: "256Mi" + +rootCoordinator: + resources: + requests: + cpu: "0.3" + memory: "256Mi" + +queryCoordinator: + resources: + requests: + cpu: "0.4" + memory: "100Mi" + +queryNode: + resources: + requests: + cpu: "2" + memory: "500Mi" +indexCoordinator: + resources: + requests: + cpu: "0.1" + memory: "50Mi" +indexNode: + resources: + requests: + cpu: "4" + memory: "500Mi" + +dataCoordinator: + resources: + requests: + cpu: "0.1" + memory: "50Mi" + +dataNode: + resources: + requests: + cpu: "0.5" + memory: "500Mi" + +pulsar: + proxy: + configData: + PULSAR_MEM: > + -Xms2048m -Xmx2048m + PULSAR_GC: > + -XX:MaxDirectMemorySize=2048m + httpNumThreads: "50" + resources: + requests: + cpu: "1" + memory: "2Gi" + # Resources for the websocket proxy + wsResources: + requests: + memory: "512Mi" + cpu: "0.3" + broker: + resources: + requests: + cpu: "1.5" + memory: "4Gi" + configData: + PULSAR_MEM: > + -Xms4096m + -Xmx4096m + -XX:MaxDirectMemorySize=8192m + PULSAR_GC: > + -Dio.netty.leakDetectionLevel=disabled + -Dio.netty.recycler.linkCapacity=1024 + -XX:+ParallelRefProcEnabled + -XX:+UnlockExperimentalVMOptions + -XX:+DoEscapeAnalysis + -XX:ParallelGCThreads=32 + -XX:ConcGCThreads=32 + -XX:G1NewSizePercent=50 + -XX:+DisableExplicitGC + -XX:-ResizePLAB + -XX:+ExitOnOutOfMemoryError + maxMessageSize: "104857600" + defaultRetentionTimeInMinutes: "10080" + defaultRetentionSizeInMB: "8192" + backlogQuotaDefaultLimitGB: "8" + backlogQuotaDefaultRetentionPolicy: producer_exception + + bookkeeper: + configData: + PULSAR_MEM: > + -Xms4096m + -Xmx4096m + -XX:MaxDirectMemorySize=8192m + PULSAR_GC: > + -Dio.netty.leakDetectionLevel=disabled + -Dio.netty.recycler.linkCapacity=1024 + -XX:+UseG1GC -XX:MaxGCPauseMillis=10 + -XX:+ParallelRefProcEnabled + -XX:+UnlockExperimentalVMOptions + -XX:+DoEscapeAnalysis + -XX:ParallelGCThreads=32 + -XX:ConcGCThreads=32 + -XX:G1NewSizePercent=50 + -XX:+DisableExplicitGC + -XX:-ResizePLAB + -XX:+ExitOnOutOfMemoryError + -XX:+PerfDisableSharedMem + -XX:+PrintGCDetails + nettyMaxFrameSizeBytes: "104867840" + resources: + requests: + cpu: 1 + memory: "4Gi" + + bastion: + resources: + requests: + cpu: "0.3" + memory: "50Mi" + + autorecovery: + resources: + requests: + cpu: "1" + memory: "512Mi" + + zookeeper: + configData: + PULSAR_MEM: > + -Xms1024m + -Xmx1024m + PULSAR_GC: > + -Dcom.sun.management.jmxremote + -Djute.maxbuffer=10485760 + -XX:+ParallelRefProcEnabled + -XX:+UnlockExperimentalVMOptions + -XX:+DoEscapeAnalysis + -XX:+DisableExplicitGC + -XX:+PerfDisableSharedMem + -Dzookeeper.forceSync=no + resources: + requests: + cpu: "0.3" + memory: "1Gi" +etcd: + resources: + requests: + cpu: "0.1" + memory: "100Mi" +minio: + resources: + requests: + cpu: "0.3" + memory: "512Mi" +standalone: + resources: + requests: + cpu: "1" + memory: "3.5Gi"