Add ci for new cluster (#18267)

Signed-off-by: Jenny Li <jing.li@zilliz.com>

.github/workflows/jenkins-checker.yaml

@@ -7,6 +7,7 @@ on:
     # file paths to consider in the event. Optional; defaults to all.
     paths:
       - 'build/ci/jenkins/**.groovy'
+      - 'ci/jenkins/**.groovy'
       - '.github/workflows/jenkins-checker.yaml'
 jobs:
   check-jenkinsfile:
@@ -18,13 +19,12 @@ jobs:
       - name: Validate Jenkinsfile
         shell: bash
         run: |
-          JENKINS_URL=https://ci.milvus.io:18080/jenkins
-          # JENKINS_CRUMB is needed if your Jenkins controller has CRSF protection enabled as it should
-          JENKINS_CRUMB=`curl "$JENKINS_URL/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,\":\",//crumb)"`
-
           function validate(){
             local file_path=${1:-Jenkinsfile}
-            response=$(curl --max-time 10 --retry 5 --retry-delay 0 --retry-max-time 40  -X POST -H $JENKINS_CRUMB -F "jenkinsfile=<${file_path}" $JENKINS_URL/pipeline-model-converter/validate)
+            local jenkins_url=${2:-"https://ci.milvus.io:18080/jenkins"}
+       
+            JENKINS_CRUMB=`curl "${jenkins_url}/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,\":\",//crumb)"`
+            response=$(curl --max-time 10 --retry 5 --retry-delay 0 --retry-max-time 40  -X POST -H $JENKINS_CRUMB -F "jenkinsfile=<${file_path}" ${jenkins_url}/pipeline-model-converter/validate)
             
             if [[ ${response} =~ "Error"  ]]
             then
@@ -34,24 +34,28 @@ jobs:
                 exit 1  
             fi
           }
+          function validate_path(){
+            local path=${1}
+            local jenkins_url=${2}
 
-          for file in build/ci/jenkins/*
-          do
-              if [ -f "$file"  ] 
-              then 
-                  echo "$file"
-                  file_name=$(basename "$file")
-
-                  if echo "${file_name}" | grep -q -E '\.groovy$'
-                  then 
-                      echo "Validate groovy file ${file_name}"
-                      validate $file
-                  elif [[ "${file_name}" == "Jenkinsfile" ]]
-                  then
-                      echo "Validate Jenkinsfile"
-                      validate $file    
-                  fi
-              fi
-          done
-          
+            for file in ${path}
+            do
+                if [ -f "$file"  ] 
+                then 
+                    # echo "$file"
+                    file_name=$(basename "$file")
+                    if echo "${file_name}" | grep -q -E '\.groovy$'
+                    then 
+                        # echo "Validate groovy file ${file_name}"
+                        validate $file ${jenkins_url}
+                    elif [[ "${file_name}" == "Jenkinsfile" ]]
+                    then
+                        # echo "Validate Jenkinsfile"
+                        validate $file ${jenkins_url} 
+                    fi
+                fi
+            done
+          }
+          validate_path "build/ci/jenkins/*" "https://ci.milvus.io:18080/jenkins/"
+          validate_path "ci/jenkins/*" "https://jenkins.milvus.io:18080/"
     

ci/jenkins/PR.groovy

@@ -0,0 +1,232 @@
+#!/usr/bin/env groovy
+
+int total_timeout_minutes = 120
+int e2e_timeout_seconds = 70 * 60
+def imageTag=''
+int case_timeout_seconds = 10 * 60
+def chart_version='3.0.27'
+pipeline {
+    options {
+        timeout(time: total_timeout_minutes, unit: 'MINUTES')
+        buildDiscarder logRotator(artifactDaysToKeepStr: '30')
+        // parallelsAlwaysFailFast()
+        preserveStashes(buildCount: 5)
+        disableConcurrentBuilds(abortPrevious: true)
+
+    }
+    agent {
+            kubernetes {
+                label 'milvus-e2e-test-pipeline'
+                inheritFrom 'default'
+                defaultContainer 'main'
+                yamlFile 'ci/jenkins/pod/rte.yaml'
+                customWorkspace '/home/jenkins/agent/workspace'
+            }
+    }
+    environment {
+        PROJECT_NAME = 'milvus'
+        SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
+        DOCKER_BUILDKIT = 1
+        ARTIFACTS = "${env.WORKSPACE}/_artifacts"
+        CI_DOCKER_CREDENTIAL_ID = "harbor-milvus-io-registry"
+        MILVUS_HELM_NAMESPACE = "milvus-ci"
+        DISABLE_KIND = true
+        HUB = 'harbor.milvus.io/milvus'
+        JENKINS_BUILD_ID = "${env.BUILD_ID}"
+        CI_MODE="pr"
+    }
+
+    stages {
+        stage ('Build'){
+            steps {
+                container('main') {
+                    dir ('build'){
+                            sh './set_docker_mirror.sh'
+                    }
+                    dir ('tests/scripts') {
+                        script {
+                            sh 'printenv'
+                            def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim()
+                            def gitShortCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()    
+                            imageTag="${env.BRANCH_NAME}-${date}-${gitShortCommit}"
+                            withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
+                                sh """
+                                TAG="${imageTag}" \
+                                ./e2e-k8s.sh \
+                                --skip-export-logs \
+                                --skip-install \
+                                --skip-cleanup \
+                                --skip-setup \
+                                --skip-test
+                                """
+
+                                // stash imageTag info for rebuild install & E2E Test only
+                                sh "echo ${imageTag} > imageTag.txt"
+                                stash includes: 'imageTag.txt', name: 'imageTag'
+
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+
+        stage('Install & E2E Test') {
+            matrix {
+                axes {
+                    axis {
+                        name 'MILVUS_SERVER_TYPE'
+                        values 'standalone', 'distributed'
+                    }
+                    axis {
+                        name 'MILVUS_CLIENT'
+                        values 'pymilvus'
+                    }
+                }
+
+                stages {
+                    stage('Install') {
+                        steps {
+                            container('main') {
+                                stash includes: 'tests/**', name: 'testCode', useDefaultExcludes: false
+                                dir ('tests/scripts') {
+                                    script {
+                                        sh 'printenv'
+                                        def clusterEnabled = "false"
+                                        if ("${MILVUS_SERVER_TYPE}" == 'distributed') {
+                                            clusterEnabled = "true"
+                                        }
+
+                                        if ("${MILVUS_CLIENT}" == "pymilvus") {
+                                            if ("${imageTag}"==''){
+                                                dir ("imageTag"){
+                                                    try{
+                                                        unstash 'imageTag'
+                                                        imageTag=sh(returnStdout: true, script: 'cat imageTag.txt | tr -d \'\n\r\'')
+                                                    }catch(e){
+                                                        print "No Image Tag info remained ,please rerun build to build new image."
+                                                        exit 1
+                                                    }
+                                                }
+                                            }
+                                            withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
+                                                sh """
+                                                MILVUS_CLUSTER_ENABLED=${clusterEnabled} \
+                                                TAG=${imageTag}\
+                                                ./e2e-k8s.sh \
+                                                --skip-export-logs \
+                                                --skip-cleanup \
+                                                --skip-setup \
+                                                --skip-test \
+                                                --skip-build \
+                                                --skip-build-image \
+                                                --install-extra-arg "
+                                                --set etcd.metrics.enabled=true \
+                                                --set etcd.metrics.podMonitor.enabled=true \
+                                                --set indexCoordinator.gc.interval=1 \
+                                                --version ${chart_version} \
+                                                -f values/ci/pr.yaml" 
+                                                """
+                                            }
+                                        } else {
+                                            error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}"
+                                        }
+                                    }
+                                }
+                            }
+
+                        }
+                    }
+                    stage('E2E Test'){
+                        options { 
+                            skipDefaultCheckout() 
+                        }
+                        agent {
+                                kubernetes {
+                                    label 'milvus-e2e-test-pr'
+                                    inheritFrom 'default'
+                                    defaultContainer 'main'
+                                    yamlFile 'ci/jenkins/pod/rte.yaml'
+                                    customWorkspace '/home/jenkins/agent/workspace'
+                                }
+                        }
+                        steps {
+                            container('pytest') {
+                                unstash('testCode')
+                                script {
+                                        sh 'ls -lah'
+                                }
+                                dir ('tests/scripts') {
+                                    script {
+                                        def release_name=sh(returnStdout: true, script: './get_release_name.sh')
+                                        def clusterEnabled = 'false'
+                                        if ("${MILVUS_SERVER_TYPE}" == "distributed") {
+                                            clusterEnabled = "true"
+                                        }
+                                        if ("${MILVUS_CLIENT}" == "pymilvus") {
+                                            sh """
+                                            MILVUS_HELM_RELEASE_NAME="${release_name}" \
+                                            MILVUS_HELM_NAMESPACE="milvus-ci" \
+                                            MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \
+                                            TEST_TIMEOUT="${e2e_timeout_seconds}" \
+                                            ./ci_e2e.sh  "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}"
+                                            """
+                            
+                                        } else {
+                                        error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}"
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                        post{
+                            always {
+                                container('pytest'){
+                                    dir("${env.ARTIFACTS}") {
+                                            sh "tar -zcvf ${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true"
+                                            archiveArtifacts artifacts: "${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ", allowEmptyArchive: true
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                post{
+                    always {
+                        container('main') {
+                            dir ('tests/scripts') {  
+                                script {
+                                    def release_name=sh(returnStdout: true, script: './get_release_name.sh')
+                                    sh "./uninstall_milvus.sh --release-name ${release_name}"
+                                    sh "./ci_logs.sh --log-dir /ci-logs  --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \
+                                    --release-name ${release_name}"
+                                    dir("${env.ARTIFACTS}") {
+                                        archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs.tar.gz", allowEmptyArchive: true
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+        }
+    }
+    post{
+        unsuccessful {
+                container('jnlp') {
+                    dir ('tests/scripts') {
+                        script {
+                            def authorEmail = sh(returnStdout: true, script: './get_author_email.sh ')
+                            emailext subject: '$DEFAULT_SUBJECT',
+                            body: '$DEFAULT_CONTENT',
+                            recipientProviders: [developers(), culprits()],
+                            replyTo: '$DEFAULT_REPLYTO',
+                            to: "${authorEmail},devops@zilliz.com"
+                        }
+                    }
+                }
+            }
+        }
+}

ci/jenkins/pod/rte.yaml

@@ -0,0 +1,75 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus-e2e
+  namespace: jenkins
+spec:
+  enableServiceLinks: false
+  containers:
+  - name: main
+    image: milvusdb/krte:20211213-dcc15e9
+    env:
+    - name: DOCKER_IN_DOCKER_ENABLED
+      value: "true"
+    - name: DOCKER_VOLUME_DIRECTORY
+      value: "/mnt/disk/.docker"
+    tty: true
+    securityContext:
+      privileged: true
+    args: ["cat"]
+    resources:
+      limits:
+        cpu: "6"
+        memory: 12Gi
+      requests:
+        cpu: "0.5"
+        memory: 5Gi
+    volumeMounts:
+    - mountPath: /docker-graph
+      name: docker-graph
+    - mountPath: /var/lib/docker
+      name: docker-root
+    - mountPath: /lib/modules
+      name: modules
+      readOnly: true
+    - mountPath: /sys/fs/cgroup
+      name: cgroup
+    - mountPath: /mnt/disk/.docker
+      name: build-cache
+      subPath: docker-volume
+    - mountPath: /ci-logs
+      name: ci-logs  
+  - name: pytest
+    image: milvusdb/pytest:20220705-e979e73
+    resources:
+      limits:
+        cpu: "6"
+        memory: 12Gi
+      requests:
+        cpu: "0.5"
+        memory: 5Gi
+    volumeMounts:
+    - mountPath: /ci-logs
+      name: ci-logs
+  volumes:
+  - emptyDir: {}
+    name: docker-graph
+  - emptyDir: {}
+    name: docker-root
+  - hostPath:
+      path: /tmp/krte/cache
+      type: DirectoryOrCreate
+    name: build-cache
+  - hostPath:
+      path: /lib/modules
+      type: Directory
+    name: modules
+  - hostPath:
+      path: /sys/fs/cgroup
+      type: Directory
+    name: cgroup
+  - name: ci-logs
+    nfs:
+      path: /ci-logs
+      server: 172.16.70.239

tests/scripts/ci-util.sh

@@ -41,7 +41,6 @@ export PIP_TRUSTED_HOST="nexus-nexus-repository-manager.nexus"
 export PIP_INDEX_URL="http://nexus-nexus-repository-manager.nexus:8081/repository/pypi-all/simple"
 export PIP_INDEX="http://nexus-nexus-repository-manager.nexus:8081/repository/pypi-all/pypi"
 export PIP_FIND_LINKS="http://nexus-nexus-repository-manager.nexus:8081/repository/pypi-all/pypi"
- python3 -m pip install --upgrade setuptools
  python3 -m pip install --no-cache-dir -r requirements.txt --timeout 30 --retries 6 
 }
 

tests/scripts/values/ci/pr.yaml

@@ -0,0 +1,166 @@
+metrics:
+  serviceMonitor:
+    enabled: true
+proxy:
+  resources:
+    requests:
+      cpu: "0.3"
+      memory: "256Mi"
+
+rootCoordinator:
+  resources:
+    requests:
+      cpu: "0.3"
+      memory: "256Mi"
+
+queryCoordinator:
+  resources:
+    requests:
+      cpu: "0.4"
+      memory: "100Mi"
+
+queryNode:
+  resources:
+    requests:
+      cpu: "2"
+      memory: "500Mi"
+indexCoordinator:
+  resources:
+    requests:
+      cpu: "0.1"
+      memory: "50Mi"
+indexNode:
+  resources:
+    requests:
+      cpu: "4"
+      memory: "500Mi"
+
+dataCoordinator:
+  resources:
+    requests:
+      cpu: "0.1"
+      memory: "50Mi"
+
+dataNode:
+  resources:
+    requests:
+      cpu: "0.5"
+      memory: "500Mi"
+
+pulsar:
+  proxy:
+    configData:
+      PULSAR_MEM: >
+        -Xms2048m -Xmx2048m
+      PULSAR_GC: >
+        -XX:MaxDirectMemorySize=2048m
+      httpNumThreads: "50"
+    resources:
+      requests:
+        cpu: "1"
+        memory: "2Gi"
+  # Resources for the websocket proxy
+    wsResources:
+      requests:
+        memory: "512Mi"
+        cpu: "0.3"
+  broker:
+    resources:
+      requests:
+        cpu: "1.5"
+        memory: "4Gi"
+    configData:
+      PULSAR_MEM: >
+        -Xms4096m
+        -Xmx4096m
+        -XX:MaxDirectMemorySize=8192m
+      PULSAR_GC: >
+        -Dio.netty.leakDetectionLevel=disabled
+        -Dio.netty.recycler.linkCapacity=1024
+        -XX:+ParallelRefProcEnabled
+        -XX:+UnlockExperimentalVMOptions
+        -XX:+DoEscapeAnalysis
+        -XX:ParallelGCThreads=32
+        -XX:ConcGCThreads=32
+        -XX:G1NewSizePercent=50
+        -XX:+DisableExplicitGC
+        -XX:-ResizePLAB
+        -XX:+ExitOnOutOfMemoryError
+      maxMessageSize: "104857600"
+      defaultRetentionTimeInMinutes: "10080"
+      defaultRetentionSizeInMB: "8192"
+      backlogQuotaDefaultLimitGB: "8"
+      backlogQuotaDefaultRetentionPolicy: producer_exception
+
+  bookkeeper:
+    configData:
+      PULSAR_MEM: >
+        -Xms4096m
+        -Xmx4096m
+        -XX:MaxDirectMemorySize=8192m
+      PULSAR_GC: >
+        -Dio.netty.leakDetectionLevel=disabled
+        -Dio.netty.recycler.linkCapacity=1024
+        -XX:+UseG1GC -XX:MaxGCPauseMillis=10
+        -XX:+ParallelRefProcEnabled
+        -XX:+UnlockExperimentalVMOptions
+        -XX:+DoEscapeAnalysis
+        -XX:ParallelGCThreads=32
+        -XX:ConcGCThreads=32
+        -XX:G1NewSizePercent=50
+        -XX:+DisableExplicitGC
+        -XX:-ResizePLAB
+        -XX:+ExitOnOutOfMemoryError
+        -XX:+PerfDisableSharedMem
+        -XX:+PrintGCDetails
+      nettyMaxFrameSizeBytes: "104867840"
+    resources:
+      requests:
+        cpu: 1
+        memory: "4Gi"
+
+  bastion:
+    resources:
+      requests:
+        cpu: "0.3"
+        memory: "50Mi"
+
+  autorecovery:
+    resources:
+      requests:
+        cpu: "1"
+        memory: "512Mi"
+
+  zookeeper:
+    configData:
+      PULSAR_MEM: >
+        -Xms1024m
+        -Xmx1024m
+      PULSAR_GC: >
+         -Dcom.sun.management.jmxremote
+         -Djute.maxbuffer=10485760
+         -XX:+ParallelRefProcEnabled
+         -XX:+UnlockExperimentalVMOptions
+         -XX:+DoEscapeAnalysis
+         -XX:+DisableExplicitGC
+         -XX:+PerfDisableSharedMem
+         -Dzookeeper.forceSync=no
+    resources:
+      requests:
+        cpu: "0.3"
+        memory: "1Gi"
+etcd:
+  resources:
+    requests:
+      cpu: "0.1"
+      memory: "100Mi"
+minio:
+  resources:
+    requests:
+      cpu: "0.3"
+      memory: "512Mi"
+standalone:
+  resources:
+    requests:
+      cpu: "1"
+      memory: "3.5Gi"