Chanege cluster for e2e to 4am (#26253)

Signed-off-by: Bennu-Li <yunmei.li@zilliz.com>

ci/jenkins/PR.groovy

@@ -17,7 +17,8 @@ pipeline {
     }
     agent {
             kubernetes {
-                inheritFrom 'milvus-e2e'
+                cloud '4am'
+                inheritFrom 'milvus-e2e-4am'
                 defaultContainer 'main'
                 yamlFile 'ci/jenkins/pod/rte-build.yaml'
                 customWorkspace '/home/jenkins/agent/workspace'
@@ -42,7 +43,9 @@ pipeline {
             steps {
                 container('main') {
                     dir ('build'){
-                            sh './set_docker_mirror.sh'
+                            sh """
+                            MIRROR_URL="https://docker-nexus-ci.zilliz.cc" ./set_docker_mirror.sh
+                            """
                     }
                     dir ('tests/scripts') {
                         script {
@@ -96,7 +99,7 @@ pipeline {
                                     script {
                                         sh 'printenv'
                                         def clusterEnabled = "false"
-                                        def valuesFile = "pr.yaml"
+                                        def valuesFile = "pr-4am.yaml"
                                         if ("${MILVUS_SERVER_TYPE}".contains('distributed')) {
                                             clusterEnabled = "true"
                                         }
@@ -125,7 +128,7 @@ pipeline {
                                                     chmod +x /usr/bin/yq
                                                 '''
                                                 sh """
-                                                    cp values/ci/pr.yaml values/ci/pr_kafka.yaml
+                                                    cp values/ci/pr-4am.yaml values/ci/pr_kafka.yaml
                                                     yq -i '.pulsar.enabled=false' values/ci/pr_kafka.yaml
                                                     yq -i '.kafka.enabled=true' values/ci/pr_kafka.yaml
                                                     yq -i '.kafka.metrics.kafka.enabled=true' values/ci/pr_kafka.yaml
@@ -136,7 +139,7 @@ pipeline {
                                             withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
                                                 sh """
                                                     MILVUS_CLUSTER_ENABLED=${clusterEnabled} \
-                                                    MILVUS_HELM_REPO="http://nexus-nexus-repository-manager.nexus:8081/repository/milvus-proxy" \
+                                                    MILVUS_HELM_REPO="https://nexus-ci.zilliz.cc/repository/milvus-proxy" \
                                                     TAG=${imageTag}\
                                                     ./e2e-k8s.sh \
                                                     --skip-export-logs \
@@ -171,6 +174,7 @@ pipeline {
                         }
                         agent {
                                 kubernetes {
+                                    cloud '4am'
                                     inheritFrom 'default'
                                     defaultContainer 'main'
                                     yamlFile 'ci/jenkins/pod/e2e.yaml'
@@ -196,7 +200,7 @@ pipeline {
                                             MILVUS_HELM_NAMESPACE="milvus-ci" \
                                             MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \
                                             TEST_TIMEOUT="${e2e_timeout_seconds}" \
-                                            ./ci_e2e.sh  "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}"
+                                            ./ci_e2e_4am.sh  "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}"
                                             """
                             
                                         } else {

ci/jenkins/pod/e2e.yaml

@@ -23,5 +23,6 @@ spec:
   volumes:
   - name: ci-logs
     nfs:
-      path: /ci-logs
+      path: /volume1/ci-logs
-      server: 172.16.70.239
+      # path: /volume1/4am-logs
+      server: 172.16.70.249

ci/jenkins/pod/rte-build.yaml

@@ -64,5 +64,6 @@ spec:
     name: cgroup
   - name: ci-logs
     nfs:
-      path: /ci-logs
+      path: /volume1/ci-logs
-      server: 172.16.70.239
+      # path: /volume1/4am-logs
+      server: 172.16.70.249

tests/scripts/ci-util-4am.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Licensed to the LF AI & Data foundation under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Exit immediately for non zero status
+set -e
+# Check unset variables
+set -u
+# Print commands
+set -x
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+ROOT="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )"
+
+
+# Install pytest requirements
+function install_pytest_requirements(){
+ echo "Install pytest requirements"
+ cd ${ROOT}/tests/python_client
+
+# MIRROR_HOST="${MIRROR_HOST:-nexus-nexus-repository-manager.nexus}"
+
+
+export PIP_TRUSTED_HOST="nexus-ci.zilliz.cc"
+export PIP_INDEX_URL="https://nexus-ci.zilliz.cc/repository/pypi-all/simple"
+export PIP_INDEX="https://nexus-ci.zilliz.cc/repository/pypi-all/pypi"
+export PIP_FIND_LINKS="https://nexus-ci.zilliz.cc/repository/pypi-all/pypi"
+ python3 -m pip install --no-cache-dir -r requirements.txt --timeout 30 --retries 6 
+}
+
+# Login in ci docker registry
+function docker_login_ci_registry(){
+
+    if [[ -z "${CI_REGISTRY_USERNAME:-}" || -z "${CI_REGISTRY_PASSWORD:-}" ]]; then 
+       echo "Please setup docker credential for ci registry-${HUB}"
+    else
+        echo "docker login ci registry"
+        docker login  -u ${CI_REGISTRY_USERNAME} -p ${CI_REGISTRY_PASSWORD} ${HUB}
+    fi
+}
+

tests/scripts/ci_e2e_4am.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Licensed to the LF AI & Data foundation under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+ROOT="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )"
+
+# Exit immediately for non zero status
+set -e
+# Check unset variables
+set -u
+# Print commands
+set -x
+
+
+
+MILVUS_HELM_RELEASE_NAME="${MILVUS_HELM_RELEASE_NAME:-milvus-testing}"
+MILVUS_CLUSTER_ENABLED="${MILVUS_CLUSTER_ENABLED:-false}"
+MILVUS_HELM_NAMESPACE="${MILVUS_HELM_NAMESPACE:-default}"
+PARALLEL_NUM="${PARALLEL_NUM:-6}"
+# Use service name instead of IP to test
+MILVUS_SERVICE_NAME=$(echo "${MILVUS_HELM_RELEASE_NAME}-milvus.${MILVUS_HELM_NAMESPACE}" | tr -d '\n')
+MILVUS_SERVICE_PORT="19530"
+# Minio service name
+MINIO_SERVICE_NAME=$(echo "${MILVUS_HELM_RELEASE_NAME}-minio.${MILVUS_HELM_NAMESPACE}" | tr -d '\n')
+
+
+# Shellcheck source=ci-util.sh
+source "${ROOT}/tests/scripts/ci-util-4am.sh"
+
+
+cd ${ROOT}/tests/python_client
+
+# Print python3 version, python version 3.6.8 is more stable for test
+python3 -V
+
+# Pytest will try to get ${CI_LOG_PATH} from environment variables first,then use default path
+export CI_LOG_PATH=/tmp/ci_logs/test
+
+if [ ! -d "${CI_LOG_PATH}" ]; then
+  # Create dir for ci log path when it does not exist
+  mkdir -p ${CI_LOG_PATH}
+fi
+
+echo "prepare e2e test"  
+install_pytest_requirements  
+
+
+# Pytest is not able to have both --timeout & --workers, so do not add --timeout or --workers in the shell script
+if [[ -n "${TEST_TIMEOUT:-}" ]]; then
+  
+  timeout  "${TEST_TIMEOUT}" pytest --host ${MILVUS_SERVICE_NAME} --port ${MILVUS_SERVICE_PORT} \
+                                     --html=${CI_LOG_PATH}/report.html  --self-contained-html ${@:-}
+else
+  pytest --host ${MILVUS_SERVICE_NAME} --port ${MILVUS_SERVICE_PORT} \
+                                     --html=${CI_LOG_PATH}/report.html --self-contained-html ${@:-}
+fi
+
+# Run bulk insert test
+if [[ -n "${TEST_TIMEOUT:-}" ]]; then
+  
+  timeout  "${TEST_TIMEOUT}" pytest testcases/test_bulk_insert.py --host ${MILVUS_SERVICE_NAME} --port ${MILVUS_SERVICE_PORT} --minio_host ${MINIO_SERVICE_NAME} \
+                                     --html=${CI_LOG_PATH}/report_bulk_insert.html  --self-contained-html
+else
+  pytest testcases/test_bulk_insert.py --host ${MILVUS_SERVICE_NAME} --port ${MILVUS_SERVICE_PORT} --minio_host ${MINIO_SERVICE_NAME} \
+                                     --html=${CI_LOG_PATH}/report_bulk_insert.html --self-contained-html
+fi

tests/scripts/ci_logs.sh

@@ -87,14 +87,19 @@ if [[ ! -d ${RELEASE_LOG_DIR} ]] ;then
 fi 
 # Try to found logs file from mount disk /volume1/ci-logs
 log_files=$(find ${LOG_DIR} -type f  -name "*${RELEASE_NAME}*" )
-for log_file in ${log_files}
-do 
- file_name=$(basename ${log_file})
-  mv  ${log_file} ${RELEASE_LOG_DIR}/`echo ${file_name} | sed 's/jenkins.var.log.containers.//g' `
-done 
 
-tar -zcvf ${ARTIFACTS_NAME:-artifacts}.tar.gz ${RELEASE_LOG_DIR}/*
+if [ -z "${log_files:-}" ]; then
-rm -rf ${RELEASE_LOG_DIR}
+  echo "No log files find"
+else
+  for log_file in ${log_files}
+  do 
+   file_name=$(basename ${log_file})
+    mv  ${log_file} ${RELEASE_LOG_DIR}/`echo ${file_name} | sed 's/jenkins.var.log.containers.//g' `
+  done 
+
+  tar -zcvf ${ARTIFACTS_NAME:-artifacts}.tar.gz ${RELEASE_LOG_DIR}/*
+  rm -rf ${RELEASE_LOG_DIR}
+fi
 
 remain_log_files=$(find ${LOG_DIR} -type f  -name "*${RELEASE_NAME}*")
 

tests/scripts/values/ci/pr-4am.yaml

@@ -0,0 +1,186 @@
+metrics:
+  serviceMonitor:
+    enabled: true
+log:
+  level: debug
+proxy:
+  resources:
+    requests:
+      cpu: "0.3"
+      memory: "256Mi"
+    limits:
+      cpu: "1"
+rootCoordinator:
+  resources:
+    requests:
+      cpu: "0.2"
+      memory: "256Mi"
+    limits:
+      cpu: "1"
+queryCoordinator:
+  resources:
+    requests:
+      cpu: "0.2"
+      memory: "100Mi"
+    limits:
+      cpu: "1"
+queryNode:
+  resources:
+    requests:
+      cpu: "0.5"
+      memory: "500Mi"
+    limits:
+      cpu: "2"
+indexCoordinator:
+  resources:
+    requests:
+      cpu: "0.1"
+      memory: "50Mi"
+    limits:
+      cpu: "1"
+indexNode:
+  resources:
+    requests:
+      cpu: "0.5"
+      memory: "500Mi"
+    limits:
+      cpu: "2"
+dataCoordinator:
+  resources:
+    requests:
+      cpu: "0.1"
+      memory: "50Mi"
+    limits:
+      cpu: "1"
+dataNode:
+  resources:
+    requests:
+      cpu: "0.5"
+      memory: "500Mi"
+    limits:
+      cpu: "2"
+
+pulsar:
+  components:
+    autorecovery: false
+  proxy:
+    configData:
+      PULSAR_MEM: >
+        -Xms1024m -Xmx1024m
+      PULSAR_GC: >
+        -XX:MaxDirectMemorySize=2048m
+      httpNumThreads: "50"
+    resources:
+      requests:
+        cpu: "0.5"
+        memory: "1Gi"
+  # Resources for the websocket proxy
+    wsResources:
+      requests:
+        memory: "100Mi"
+        cpu: "0.1"
+  broker:
+    resources:
+      requests:
+        cpu: "0.5"
+        memory: "4Gi"
+    configData:
+      PULSAR_MEM: >
+        -Xms4096m
+        -Xmx4096m
+        -XX:MaxDirectMemorySize=8192m
+      PULSAR_GC: >
+        -Dio.netty.leakDetectionLevel=disabled
+        -Dio.netty.recycler.linkCapacity=1024
+        -XX:+ParallelRefProcEnabled
+        -XX:+UnlockExperimentalVMOptions
+        -XX:+DoEscapeAnalysis
+        -XX:ParallelGCThreads=32
+        -XX:ConcGCThreads=32
+        -XX:G1NewSizePercent=50
+        -XX:+DisableExplicitGC
+        -XX:-ResizePLAB
+        -XX:+ExitOnOutOfMemoryError
+      maxMessageSize: "104857600"
+      defaultRetentionTimeInMinutes: "10080"
+      defaultRetentionSizeInMB: "8192"
+      backlogQuotaDefaultLimitGB: "8"
+      backlogQuotaDefaultRetentionPolicy: producer_exception
+
+  bookkeeper:
+    configData:
+      PULSAR_MEM: >
+        -Xms4096m
+        -Xmx4096m
+        -XX:MaxDirectMemorySize=8192m
+      PULSAR_GC: >
+        -Dio.netty.leakDetectionLevel=disabled
+        -Dio.netty.recycler.linkCapacity=1024
+        -XX:+UseG1GC -XX:MaxGCPauseMillis=10
+        -XX:+ParallelRefProcEnabled
+        -XX:+UnlockExperimentalVMOptions
+        -XX:+DoEscapeAnalysis
+        -XX:ParallelGCThreads=32
+        -XX:ConcGCThreads=32
+        -XX:G1NewSizePercent=50
+        -XX:+DisableExplicitGC
+        -XX:-ResizePLAB
+        -XX:+ExitOnOutOfMemoryError
+        -XX:+PerfDisableSharedMem
+        -XX:+PrintGCDetails
+      nettyMaxFrameSizeBytes: "104867840"
+    resources:
+      requests:
+        cpu: "0.5"
+        memory: "4Gi"
+
+  zookeeper:
+    replicaCount: 1
+    configData:
+      PULSAR_MEM: >
+        -Xms1024m
+        -Xmx1024m
+      PULSAR_GC: >
+         -Dcom.sun.management.jmxremote
+         -Djute.maxbuffer=10485760
+         -XX:+ParallelRefProcEnabled
+         -XX:+UnlockExperimentalVMOptions
+         -XX:+DoEscapeAnalysis
+         -XX:+DisableExplicitGC
+         -XX:+PerfDisableSharedMem
+         -Dzookeeper.forceSync=no
+    resources:
+      requests:
+        cpu: "0.3"
+        memory: "512Mi"
+kafka:
+  resources:
+    requests:
+      cpu: "0.5"
+      memory: "1Gi"
+  zookeeper:
+    replicaCount: 1
+    resources:
+      requests:
+        cpu: "0.3"
+        memory: "512Mi"
+etcd:
+  nodeSelector:
+    nvme: "true"
+  replicaCount: 1
+  resources:
+    requests:
+      cpu: "0.3"
+      memory: "100Mi"
+minio:
+  resources:
+    requests:
+      cpu: "0.3"
+      memory: "512Mi"
+standalone:
+  resources:
+    requests:
+      cpu: "1"
+      memory: "3.5Gi"
+    limits:
+      cpu: "4"