[skip ci]Update chaos test (#12206)

Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>

.github/workflows/pod-failure-chaos-test.yaml

@@ -86,7 +86,7 @@ jobs:
           sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_pod_failure.yaml\'/g" constants.py
           sed -i "s/CHAOS_DURATION =.*/CHAOS_DURATION = 80/g" constants.py
           cat constants.py
-          pytest -s -v test_chaos.py --host 127.0.0.1
+          pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO
 
       - name: Milvus E2E Test
         timeout-minutes: 5
@@ -95,6 +95,8 @@ jobs:
         working-directory: tests/python_client/chaos
         run: |
           kubectl get pod -n chaos-testing
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=milvus-chaos -n chaos-testing --timeout=360s
+          kubectl wait --for=condition=Ready pod -l release=milvus-chaos -n chaos-testing --timeout=360s          
           python hello_milvus.py
 
       - name: Data Consist Test
@@ -103,7 +105,7 @@ jobs:
         shell: bash
         working-directory: tests/python_client/chaos
         run: |
-          pytest -s -v test_chaos_data_consist.py --host 127.0.0.1
+          pytest -s -v test_chaos_data_consist.py --host 127.0.0.1 --log-cli-level=INFO
 
       - name: Milvus E2E Test
         timeout-minutes: 5
@@ -112,6 +114,8 @@ jobs:
         working-directory: tests/python_client/chaos
         run: |
           kubectl get pod -n chaos-testing
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=milvus-chaos -n chaos-testing --timeout=360s
+          kubectl wait --for=condition=Ready pod -l release=milvus-chaos -n chaos-testing --timeout=360s          
           python hello_milvus.py
 
       - name: Export logs

.github/workflows/pod-kill-chaos-test.yaml

@@ -85,13 +85,13 @@ jobs:
           sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/pod_kill\/'/g" constants.py
           sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_pod_kill.yaml\'/g" constants.py
           cat constants.py
-          timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 || echo "chaos test failed"
+          timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO || echo "chaos test failed"
 
       - name: Milvus E2E Test
         timeout-minutes: 10
         if: ${{ always() }}
         shell: bash
-        working-directory: tests/python_client/chaos
+        working-directory: tests/python_client
         run: |
           kubectl get pod -n chaos-testing
           # wait all pod to be ready
@@ -104,7 +104,8 @@ jobs:
           sleep 20s
           nc -vz 127.0.0.1 19530
 
-          python scripts/hello_milvus.py
+          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO
+          python chaos/scripts/hello_milvus.py --host 127.0.0.1
       
       - name: Deploy Milvus Again If Previous E2E Test Failed
         timeout-minutes: 15
@@ -133,21 +134,25 @@ jobs:
         shell: bash
         working-directory: tests/python_client/chaos
         run: |
-          timeout 4m pytest -s -v test_chaos_data_consist.py --host 127.0.0.1 || echo "data consist test failed"
+          pytest -s -v test_chaos_data_consist.py --host 127.0.0.1 --log-cli-level=INFO
 
       - name: Milvus E2E Test
-        timeout-minutes: 5
+        timeout-minutes: 10
         if: ${{ always() }}
         shell: bash
-        working-directory: tests/python_client/chaos
+        working-directory: tests/python_client
         run: |
+          kubectl get pod -n chaos-testing
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=milvus-chaos -n chaos-testing --timeout=360s
+          kubectl wait --for=condition=Ready pod -l release=milvus-chaos -n chaos-testing --timeout=360s
           kubectl get pod -n chaos-testing       
           ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9  
           kubectl port-forward service/milvus-chaos 19530 -n chaos-testing >/dev/null 2>&1 &
           sleep 20s
           nc -vz 127.0.0.1 19530
 
-          python scripts/hello_milvus.py
+          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO
+          python chaos/scripts/hello_milvus.py --host 127.0.0.1
 
       - name: Export logs
         if: ${{ always() }}

tests/python_client/chaos/chaos_test.sh

@@ -12,17 +12,17 @@ elif [[ "$unamestr" == 'Darwin' ]]; then
 fi
 echo "platform: $platform"
 
-# define chaos testing object
+release="milvus-chaos"
-release=${1:-"milvus-chaos"}
+ns="chaos-testing"
-ns=${2:-"chaos-testing"}
 
 # switch namespace
 kubectl config set-context --current --namespace=${ns}
-pod="standalone"
+
-chaos_type="pod_kill"
+# set parameters
-chaos_task="data-consist-test" # chaos-test or data-consist-test 
+pod=${1:-"querynode"}
-release="milvus-chaos"
+chaos_type=${2:-"pod_kill"}
-ns="chaos-testing"
+chaos_task=${3:-"chaos-test"} # chaos-test or data-consist-test 
+
 
 # install milvus cluster for chaos testing
 pushd ./scripts
@@ -66,16 +66,20 @@ python scripts/hello_milvus.py --host "$host"
 # chaos test
 if [ "$chaos_task" == "chaos-test" ];
 then
-    pytest -s -v test_chaos.py --host "$host" || echo "chaos test fail"
+    pytest -s -v test_chaos.py --host "$host" --log-cli-level=INFO || echo "chaos test fail"
 fi
 # data consist test
 if [ "$chaos_task" == "data-consist-test" ];
 then
-    pytest -s -v test_chaos_data_consist.py --host "$host" || echo "chaos test fail"
+    pytest -s -v test_chaos_data_consist.py --host "$host" --log-cli-level=INFO || echo "chaos test fail"
 fi
 sleep 30s
 echo "start running e2e test"
+kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=milvus-chaos -n chaos-testing --timeout=360s
+kubectl wait --for=condition=Ready pod -l release=milvus-chaos -n chaos-testing --timeout=360s
+
 python scripts/hello_milvus.py --host "$host" || echo "e2e test fail"
 
 # save logs
-bash ../../scripts/export_log_k8s.sh ${ns} ${release} k8s_log/${pod}
+data=`date +%Y-%m-%d-%H-%M-%S`
+bash ../../scripts/export_log_k8s.sh ${ns} ${release} k8s_log/${pod}-${chaos_type}-${data}

tests/python_client/chaos/run.sh

@@ -0,0 +1,8 @@
+
+
+pods=("standalone" "datacoord" "proxy" "pulsar" "querynode" "rootcoord" "etcd")
+for pod in ${pods[*]}
+do
+echo "run pod kill chaos test for pod $pod "
+bash chaos_test.sh $pod
+done

tests/python_client/chaos/scripts/hello_milvus.py

@@ -98,7 +98,7 @@ def hello_milvus(host="127.0.0.1"):
     sorted_res = sorted(res, key=lambda k: k['count'])
     for r in sorted_res:
         print(r)
-
+    collection.release()
 
 import argparse