admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-08 10:08:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

[skip e2e]Refactor chaos test to support multi experiments (#14704)

Browse Source 
Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>
This commit is contained in:
zhuwenxing 2022-01-04 09:03:20 +08:00 committed by GitHub
parent a0984c2c8e
commit db1479428e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 13 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/pod-failure-chaos-test.yaml vendored
View File

@ -93,6 +93,7 @@ jobs:
sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${{ matrix.chaos_type }}\/'/g" constants.py sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${{ matrix.chaos_type }}\/'/g" constants.py
sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_${{ matrix.chaos_type }}.yaml\'/g" constants.py sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_${{ matrix.chaos_type }}.yaml\'/g" constants.py
sed -i "s/CHAOS_DURATION =.*/CHAOS_DURATION = 80/g" constants.py sed -i "s/CHAOS_DURATION =.*/CHAOS_DURATION = 80/g" constants.py
sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py
cat constants.py cat constants.py
pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no

1
.github/workflows/pod-kill-chaos-test.yaml vendored
View File

@ -88,6 +88,7 @@ jobs:
# replace chaos object # replace chaos object
sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/pod_kill\/'/g" constants.py sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/pod_kill\/'/g" constants.py
sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_pod_kill.yaml\'/g" constants.py sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_pod_kill.yaml\'/g" constants.py
sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py
cat constants.py cat constants.py
timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed" timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed"

5
tests/python_client/chaos/chaos_test.sh
View File

@ -23,7 +23,8 @@ chaos_type=${2:-"pod_kill"} #pod_kill or pod_failure
chaos_task=${3:-"chaos-test"} # chaos-test or data-consist-test chaos_task=${3:-"chaos-test"} # chaos-test or data-consist-test
node_num=${4:-1} # cluster_1_node or cluster_n_nodes node_num=${4:-1} # cluster_1_node or cluster_n_nodes
release="test"-${pod}-${chaos_type/_/-} # replace pod_kill to pod-kill cur_time=`date +%H-%M-%S`
release="test"-${pod}-${chaos_type/_/-}-${cur_time} # replace pod_kill to pod-kill
# install milvus cluster for chaos testing # install milvus cluster for chaos testing
pushd ./scripts pushd ./scripts
@ -55,9 +56,11 @@ if [ "$platform" == "Mac" ];
then then
sed -i "" "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${chaos_type}\/'/g" constants.py sed -i "" "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${chaos_type}\/'/g" constants.py
sed -i "" "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${pod}_${chaos_type}.yaml\'/g" constants.py sed -i "" "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${pod}_${chaos_type}.yaml\'/g" constants.py
sed -i "" "s/RELEASE_NAME =.*/RELEASE_NAME = \'${release}\'/g" constants.py
else else
sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${chaos_type}\/'/g" constants.py sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${chaos_type}\/'/g" constants.py
sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${pod}_${chaos_type}.yaml\'/g" constants.py sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${pod}_${chaos_type}.yaml\'/g" constants.py
sed -i "" "s/RELEASE_NAME =.*/RELEASE_NAME = \'${release}\'/g" constants.py
fi fi
# run chaos testing # run chaos testing

5
tests/python_client/chaos/constants.py
View File

@ -16,8 +16,9 @@ DELTA_PER_INS = 10 # entities per insert
ENTITIES_FOR_SEARCH = 3000 # entities for search_collection ENTITIES_FOR_SEARCH = 3000 # entities for search_collection
CHAOS_CONFIG_ENV = 'CHAOS_CONFIG_PATH' # env variables for chao path CHAOS_CONFIG_ENV = 'CHAOS_CONFIG_PATH' # env variables for chao path
TESTS_CONFIG_LOCATION = 'chaos_objects/pod_kill/' # path to the chaos CRD TESTS_CONFIG_LOCATION = 'chaos_objects/pod_kill/'
ALL_CHAOS_YAMLS = 'chaos_datanode*.yaml' # chaos file name(s) to be run against ALL_CHAOS_YAMLS = 'chaos_querynode_pod_kill.yaml'
RELEASE_NAME = 'test-querynode-pod-kill-17-33-50'
WAIT_PER_OP = 10 # time to wait in seconds between operations WAIT_PER_OP = 10 # time to wait in seconds between operations
CHAOS_DURATION = 120 # chaos duration time in seconds CHAOS_DURATION = 120 # chaos duration time in seconds
DEFAULT_INDEX_PARAM = {"index_type": "IVF_SQ8", "metric_type": "L2", "params": {"nlist": 64}} DEFAULT_INDEX_PARAM = {"index_type": "IVF_SQ8", "metric_type": "L2", "params": {"nlist": 64}}

5
tests/python_client/chaos/test_chaos.py
View File

@ -140,13 +140,16 @@ class TestChaos(TestChaosBase):
def test_chaos(self, chaos_yaml): def test_chaos(self, chaos_yaml):
# start the monitor threads to check the milvus ops # start the monitor threads to check the milvus ops
log.info("*********************Chaos Test Start**********************") log.info("*********************Chaos Test Start**********************")
log.info(f"chaos_yaml: {chaos_yaml}")
log.info(connections.get_connection_addr('default')) log.info(connections.get_connection_addr('default'))
cc.start_monitor_threads(self.health_checkers) cc.start_monitor_threads(self.health_checkers)
# parse chaos object # parse chaos object
chaos_config = cc.gen_experiment_config(chaos_yaml) chaos_config = cc.gen_experiment_config(chaos_yaml)
release_name = constants.RELEASE_NAME
log.info(f"release_name: {release_name}")
chaos_config['metadata']['name'] = release_name
meta_name = chaos_config.get('metadata', None).get('name', None) meta_name = chaos_config.get('metadata', None).get('name', None)
release_name = meta_name
chaos_config_str = json.dumps(chaos_config) chaos_config_str = json.dumps(chaos_config)
chaos_config_str = chaos_config_str.replace("milvus-chaos", release_name) chaos_config_str = chaos_config_str.replace("milvus-chaos", release_name)
chaos_config = json.loads(chaos_config_str) chaos_config = json.loads(chaos_config_str)