diff --git a/.github/workflows/pod-failure-chaos-test.yaml b/.github/workflows/pod-failure-chaos-test.yaml index e482fd42ea..53bc372f9c 100644 --- a/.github/workflows/pod-failure-chaos-test.yaml +++ b/.github/workflows/pod-failure-chaos-test.yaml @@ -93,6 +93,7 @@ jobs: sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${{ matrix.chaos_type }}\/'/g" constants.py sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_${{ matrix.chaos_type }}.yaml\'/g" constants.py sed -i "s/CHAOS_DURATION =.*/CHAOS_DURATION = 80/g" constants.py + sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py cat constants.py pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no diff --git a/.github/workflows/pod-kill-chaos-test.yaml b/.github/workflows/pod-kill-chaos-test.yaml index e514ffe199..46bb441901 100644 --- a/.github/workflows/pod-kill-chaos-test.yaml +++ b/.github/workflows/pod-kill-chaos-test.yaml @@ -88,6 +88,7 @@ jobs: # replace chaos object sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/pod_kill\/'/g" constants.py sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_pod_kill.yaml\'/g" constants.py + sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py cat constants.py timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed" diff --git a/tests/python_client/chaos/chaos_test.sh b/tests/python_client/chaos/chaos_test.sh index 474790f3ec..8df5a71a42 100644 --- a/tests/python_client/chaos/chaos_test.sh +++ b/tests/python_client/chaos/chaos_test.sh @@ -23,7 +23,8 @@ chaos_type=${2:-"pod_kill"} #pod_kill or pod_failure chaos_task=${3:-"chaos-test"} # chaos-test or data-consist-test node_num=${4:-1} # cluster_1_node or cluster_n_nodes -release="test"-${pod}-${chaos_type/_/-} # replace pod_kill to pod-kill +cur_time=`date +%H-%M-%S` +release="test"-${pod}-${chaos_type/_/-}-${cur_time} # replace pod_kill to pod-kill # install milvus cluster for chaos testing pushd ./scripts @@ -55,9 +56,11 @@ if [ "$platform" == "Mac" ]; then sed -i "" "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${chaos_type}\/'/g" constants.py sed -i "" "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${pod}_${chaos_type}.yaml\'/g" constants.py + sed -i "" "s/RELEASE_NAME =.*/RELEASE_NAME = \'${release}\'/g" constants.py else sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${chaos_type}\/'/g" constants.py sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${pod}_${chaos_type}.yaml\'/g" constants.py + sed -i "" "s/RELEASE_NAME =.*/RELEASE_NAME = \'${release}\'/g" constants.py fi # run chaos testing diff --git a/tests/python_client/chaos/constants.py b/tests/python_client/chaos/constants.py index 12b67cb321..fb0a3cfe97 100644 --- a/tests/python_client/chaos/constants.py +++ b/tests/python_client/chaos/constants.py @@ -16,8 +16,9 @@ DELTA_PER_INS = 10 # entities per insert ENTITIES_FOR_SEARCH = 3000 # entities for search_collection CHAOS_CONFIG_ENV = 'CHAOS_CONFIG_PATH' # env variables for chao path -TESTS_CONFIG_LOCATION = 'chaos_objects/pod_kill/' # path to the chaos CRD -ALL_CHAOS_YAMLS = 'chaos_datanode*.yaml' # chaos file name(s) to be run against +TESTS_CONFIG_LOCATION = 'chaos_objects/pod_kill/' +ALL_CHAOS_YAMLS = 'chaos_querynode_pod_kill.yaml' +RELEASE_NAME = 'test-querynode-pod-kill-17-33-50' WAIT_PER_OP = 10 # time to wait in seconds between operations CHAOS_DURATION = 120 # chaos duration time in seconds DEFAULT_INDEX_PARAM = {"index_type": "IVF_SQ8", "metric_type": "L2", "params": {"nlist": 64}} diff --git a/tests/python_client/chaos/test_chaos.py b/tests/python_client/chaos/test_chaos.py index 7f8500cc69..83fa7b87c6 100644 --- a/tests/python_client/chaos/test_chaos.py +++ b/tests/python_client/chaos/test_chaos.py @@ -140,13 +140,16 @@ class TestChaos(TestChaosBase): def test_chaos(self, chaos_yaml): # start the monitor threads to check the milvus ops log.info("*********************Chaos Test Start**********************") + log.info(f"chaos_yaml: {chaos_yaml}") log.info(connections.get_connection_addr('default')) cc.start_monitor_threads(self.health_checkers) # parse chaos object chaos_config = cc.gen_experiment_config(chaos_yaml) + release_name = constants.RELEASE_NAME + log.info(f"release_name: {release_name}") + chaos_config['metadata']['name'] = release_name meta_name = chaos_config.get('metadata', None).get('name', None) - release_name = meta_name chaos_config_str = json.dumps(chaos_config) chaos_config_str = chaos_config_str.replace("milvus-chaos", release_name) chaos_config = json.loads(chaos_config_str)