From e48b0fb5a8d3c61ae429b0bbe1892ae4ff24e94c Mon Sep 17 00:00:00 2001 From: Jenny Li Date: Tue, 18 Jan 2022 17:47:36 +0800 Subject: [PATCH] Add Resource limit for PR CI (#15274) Signed-off-by: Jenny Li --- build/ci/jenkins/PR.groovy | 3 +- tests/scripts/install_milvus.sh | 12 +++ tests/scripts/uninstall_milvus.sh | 13 ++- tests/scripts/values/pr.yaml | 151 ++++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 tests/scripts/values/pr.yaml diff --git a/build/ci/jenkins/PR.groovy b/build/ci/jenkins/PR.groovy index 18a55b3cfc..903c38f17a 100644 --- a/build/ci/jenkins/PR.groovy +++ b/build/ci/jenkins/PR.groovy @@ -126,7 +126,8 @@ pipeline { --set etcd.metrics.enabled=true \ --set etcd.metrics.podMonitor.enabled=true \ --set etcd.nodeSelector.disk=fast \ - --set metrics.serviceMonitor.enabled=true" + --set metrics.serviceMonitor.enabled=true \ + -f values/pr.yaml" """ } } else { diff --git a/tests/scripts/install_milvus.sh b/tests/scripts/install_milvus.sh index 7db035ff33..fdc0db285b 100755 --- a/tests/scripts/install_milvus.sh +++ b/tests/scripts/install_milvus.sh @@ -96,5 +96,17 @@ fi exitcode=$? # List pod list & pvc list before exit after helm install kubectl get pods -n ${MILVUS_HELM_NAMESPACE} -o wide | grep "${MILVUS_HELM_RELEASE_NAME}-" + + +restart_pods=$(kubectl get pods -n ${MILVUS_HELM_NAMESPACE} | grep "${MILVUS_HELM_RELEASE_NAME}-" | grep 'ago)' | awk '{print $1}') + +for restart_pod in ${restart_pods} +do + reason=$(kubectl get pod ${restart_pod} -n milvus-ci -o json | jq .status.containerStatuses[0].lastState.terminated.reason ) + restart_count=$(kubectl get pod ${restart_pod} -n milvus-ci -o json | jq .status.containerStatuses[0].restartCount ) + echo "${restart_pod} restarts ${restart_count}, last terminateed reason is ${reason}" +done + + kubectl get pvc -n ${MILVUS_HELM_NAMESPACE} | grep "${MILVUS_HELM_RELEASE_NAME}-" | awk '{$3=null;print $0}' exit ${exitcode} diff --git a/tests/scripts/uninstall_milvus.sh b/tests/scripts/uninstall_milvus.sh index 2e30e18cd9..c5197ee008 100755 --- a/tests/scripts/uninstall_milvus.sh +++ b/tests/scripts/uninstall_milvus.sh @@ -14,7 +14,7 @@ # Exit immediately for non zero status set -e # Print commands -set -x +# set -x while (( "$#" )); do case "$1" in @@ -57,7 +57,16 @@ if [[ -n "${RELEASE_NAME:-}" ]]; then MILVUS_HELM_RELEASE_NAME="${RELEASE_NAME}" # List pod list before uninstall kubectl get pods -n ${MILVUS_HELM_NAMESPACE} -o wide | grep "${MILVUS_HELM_RELEASE_NAME}-" - + # Show restart pods last terminated reason + restart_pods=$(kubectl get pods -n ${MILVUS_HELM_NAMESPACE} | grep "${MILVUS_HELM_RELEASE_NAME}-" | grep 'ago)' | awk '{print $1}') + + for restart_pod in ${restart_pods} + do + reason=$(kubectl get pod ${restart_pod} -n milvus-ci -o json | jq .status.containerStatuses[0].lastState.terminated.reason ) + restart_count=$(kubectl get pod ${restart_pod} -n milvus-ci -o json | jq .status.containerStatuses[0].restartCount ) + echo "${restart_pod} restarts ${restart_count}, last terminateed reason is ${reason}" + done + fi # Uninstall Milvus Helm Release diff --git a/tests/scripts/values/pr.yaml b/tests/scripts/values/pr.yaml new file mode 100644 index 0000000000..559bedad53 --- /dev/null +++ b/tests/scripts/values/pr.yaml @@ -0,0 +1,151 @@ + +proxy: + resources: + requests: + cpu: "0.3" + memory: "256Mi" + limits: + cpu: "0.5" + memory: "512Mi" + +rootCoordinator: + resources: + requests: + cpu: "0.3" + memory: "256Mi" + limits: + cpu: "0.6" + memory: "512Mi" + +queryCoordinator: + resources: + requests: + cpu: "0.4" + memory: "100Mi" + limits: + cpu: "1.1" + memory: "256Mi" + +queryNode: + resources: + requests: + cpu: "0.2" + memory: "500Mi" + limits: + cpu: "0.5" + memory: "1.5Gi" +indexCoordinator: + resources: + requests: + cpu: "0.1" + memory: "50Mi" + limits: + cpu: "0.1" + memory: "100Mi" +indexNode: + resources: + requests: + cpu: "0.5" + memory: "500Mi" + limits: + cpu: "1.1" + memory: "800Mi" + +dataCoordinator: + resources: + requests: + cpu: "0.1" + memory: "50Mi" + limits: + cpu: "0.2" + memory: "100Mi" + +dataNode: + resources: + requests: + cpu: "0.5" + memory: "500Mi" + limits: + cpu: "1.2" + memory: "1.5Gi" +pulsar: + proxy: + resources: + requests: + cpu: "1" + memory: "2Gi" + limits: + cpu: "1" + memory: "4Gi" + # Resources for the websocket proxy + wsResources: + requests: + memory: "512Mi" + cpu: "0.3" + limits: + memory: "512Mi" + cpu: "0.3" + broker: + resources: + requests: + cpu: "1.5" + memory: "4Gi" + limits: + cpu: "1.5" + memory: "8Gi" + bookkeeper: + resources: + requests: + cpu: 1 + memory: "4Gi" + limits: + cpu: 1 + memory: "8Gi" + bastion: + resources: + requests: + cpu: "0.3" + memory: "50Mi" + limits: + cpu: "0.3" + memory: "50Mi" + autorecovery: + resources: + requests: + cpu: "1" + memory: "512Mi" + limits: + cpu: "1" + memory: "1Gi" + zookeeper: + resources: + requests: + cpu: "0.3" + memory: "1Gi" + limits: + cpu: "0.3" + memory: "2Gi" +etcd: + resources: + requests: + cpu: "0.1" + memory: "100Mi" + limits: + cpu: "0.1" + memory: "300Mi" +minio: + resources: + requests: + cpu: "0.3" + memory: "512Mi" + limits: + cpu: "0.6" + memory: "512Mi" +standalone: + resources: + requests: + cpu: "1" + memory: "3.5Gi" + limits: + cpu: "2.6" + memory: "8.5Gi" \ No newline at end of file