diff --git a/tests/python_client/scale/scale_common.py b/tests/python_client/scale/scale_common.py index ae8481086e..d102a600dd 100644 --- a/tests/python_client/scale/scale_common.py +++ b/tests/python_client/scale/scale_common.py @@ -2,6 +2,7 @@ import os from pymilvus import connections, Index +from customize.milvus_operator import MilvusOperator from scale import constants from utils.util_log import test_log as log from base.collection_wrapper import ApiCollectionWrapper @@ -25,6 +26,31 @@ def get_milvus_chart_env_var(var=constants.MILVUS_CHART_ENV): # return milvus_helm_chart +def deploy_default_milvus(release_name, image_tag=None): + if image_tag is None: + image = f'{constants.IMAGE_REPO}:{constants.IMAGE_TAG}' + else: + image = f'{constants.IMAGE_REPO}:{image_tag}' + default_config = { + 'metadata.namespace': constants.NAMESPACE, + 'metadata.name': release_name, + 'spec.components.image': image, + 'spec.components.proxy.serviceType': 'LoadBalancer' + } + + milvusOp = MilvusOperator() + milvusOp.install(default_config) + + if milvusOp.wait_for_healthy(release_name, namespace=constants.NAMESPACE): + endpoint = milvusOp.endpoint(release_name, constants.NAMESPACE) + endpoint = endpoint.split(':') + host = endpoint[0] + port = int(endpoint[-1]) + return milvusOp, host, port + else: + raise Exception(f"Failed to install {release_name}") + + def e2e_milvus(host, c_name, collection_exist=False): # connect connections.add_connection(default={"host": host, "port": 19530}) diff --git a/tests/python_client/scale/test_data_node_scale.py b/tests/python_client/scale/test_data_node_scale.py index b367b7874b..121a3a8259 100644 --- a/tests/python_client/scale/test_data_node_scale.py +++ b/tests/python_client/scale/test_data_node_scale.py @@ -2,12 +2,10 @@ import pytest from base.collection_wrapper import ApiCollectionWrapper from common.common_type import CaseLabel -from utils.util_log import test_log as log from common import common_func as cf from common import common_type as ct -from scale import constants +from scale import constants, scale_common from scale.helm_env import HelmEnv -from customize.milvus_operator import MilvusOperator from pymilvus import connections, utility prefix = "data_scale" @@ -28,29 +26,8 @@ class TestDataNodeScale: expected: two collection create and insert op are both correctly """ release_name = "scale-data" + milvusOp, host, port = scale_common.deploy_default_milvus(release_name) - # deploy cluster milvus with dataNode 1 replicas - default_config = { - 'metadata.namespace': constants.NAMESPACE, - 'metadata.name': release_name, - 'spec.components.image': 'milvusdb/milvus-dev:master-20211020-b40513b', - 'spec.components.proxy.serviceType': 'LoadBalancer', - 'dependencies.etcd.inCluster.deletionPolicy': 'Delete', - 'dependencies.etcd.inCluster.pvcDeletion': 'true', - 'dependencies.pulsar.inCluster.deletionPolicy': 'Delete', - 'dependencies.pulsar.inCluster.pvcDeletion': 'true', - 'dependencies.storage.inCluster.deletionPolicy': 'Delete', - 'dependencies.storage.inCluster.pvcDeletion': 'true', - } - milvusOp = MilvusOperator() - milvusOp.install(default_config) - if milvusOp.wait_for_healthy(release_name, namespace=constants.NAMESPACE): - endpoint = milvusOp.endpoint(release_name, constants.NAMESPACE) - endpoint = endpoint.split(':') - host = endpoint[0] - port = int(endpoint[-1]) - else: - raise Exception(f"Failed to install {release_name}") # connect connections.add_connection(default={"host": host, "port": port}) @@ -60,11 +37,12 @@ class TestDataNodeScale: collection_w = ApiCollectionWrapper() collection_w.init_collection(name=c_name, schema=cf.gen_default_collection_schema()) # # insert - data = cf.gen_default_list_data(ct.default_nb) + data = cf.gen_default_list_data() mutation_res, _ = collection_w.insert(data) assert mutation_res.insert_count == ct.default_nb # scale dataNode to 2 pods milvusOp.upgrade(release_name, {'spec.components.dataNode.replicas': 2}, constants.NAMESPACE) + milvusOp.wait_for_healthy(release_name, constants.NAMESPACE) # after scale, assert data consistent assert utility.has_collection(c_name) diff --git a/tests/python_client/scale/test_index_node_scale.py b/tests/python_client/scale/test_index_node_scale.py index 6a3f34f060..184e890436 100644 --- a/tests/python_client/scale/test_index_node_scale.py +++ b/tests/python_client/scale/test_index_node_scale.py @@ -6,6 +6,8 @@ from pymilvus import connections from base.collection_wrapper import ApiCollectionWrapper from common.common_type import CaseLabel from scale.helm_env import HelmEnv +from scale import constants +from scale import scale_common from common import common_func as cf from common import common_type as ct from utils.util_log import test_log as log @@ -27,11 +29,10 @@ class TestIndexNodeScale: expected: The cost of one indexNode is about twice that of two indexNodes """ release_name = "scale-index" - env = HelmEnv(release_name=release_name) - host = env.helm_install_cluster_milvus() + milvusOp, host, port = scale_common.deploy_default_milvus(release_name) # connect - connections.add_connection(default={"host": host, "port": 19530}) + connections.add_connection(default={"host": host, "port": port}) connections.connect(alias='default') data = cf.gen_default_dataframe_data(nb) @@ -59,7 +60,8 @@ class TestIndexNodeScale: assert not collection_w.has_index()[0] # expand indexNode from 1 to 2 - env.helm_upgrade_cluster_milvus(indexNode=2) + milvusOp.upgrade(release_name, {'spec.components.indexNode.replicas': 2}, constants.NAMESPACE) + milvusOp.wait_for_healthy(release_name, constants.NAMESPACE) start = datetime.datetime.now() collection_w.create_index(ct.default_float_vec_field_name, default_index_params) @@ -69,6 +71,8 @@ class TestIndexNodeScale: log.debug(f't1: {t1}') assert round(t0 / t1) == 2 + # milvusOp.uninstall(release_name, namespace=constants.NAMESPACE) + @pytest.mark.tags(CaseLabel.L3) def test_shrink_index_node(self): """