admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

[skip e2e]Fix deploy test (#20649)

Browse Source 
Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>

Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>
This commit is contained in:
zhuwenxing 2022-11-16 16:51:08 +08:00 committed by GitHub
parent f76520140d
commit 45205da8f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 47 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
tests/python_client/deploy/testcases/test_action_first_deployment.py
View File

@ -1,4 +1,5 @@
import pytest
import pymilvus
from common import common_func as cf
from common import common_type as ct
from common.common_type import CaseLabel, CheckTasks
@ -8,6 +9,7 @@ from deploy.base import TestDeployBase
from deploy.common import gen_index_param, gen_search_param
from utils.util_log import test_log as log
pymilvus_version = pymilvus.__version__
default_nb = ct.default_nb
default_nq = ct.default_nq
@ -25,6 +27,8 @@ default_term_expr = f'{ct.default_int64_field_name} in [0, 1]'
prefix = "deploy_test"
TIMEOUT = 120
class TestActionFirstDeployment(TestDeployBase):
""" Test case of action before reinstall """
@ -51,24 +55,26 @@ class TestActionFirstDeployment(TestDeployBase):
is_binary = False
if "BIN" in name:
is_binary = True
self.init_collection_general(insert_data=False, is_binary=is_binary, name=name)[0]
collection_w = self.init_collection_general(insert_data=False, is_binary=is_binary, name=name)[0]
if collection_w.has_index():
index_names = [index.index_name for index in collection_w.indexes]
for index_name in index_names:
collection_w.drop_index(index_name=index_name)
@pytest.mark.tags(CaseLabel.L3)
@pytest.mark.parametrize("replica_number", [0, 1, 2])
@pytest.mark.parametrize("is_compacted", ["is_compacted", "not_compacted"])
@pytest.mark.parametrize("is_deleted", ["is_deleted"])
@pytest.mark.parametrize("is_string_indexed", ["is_string_indexed", "not_string_indexed"])
@pytest.mark.parametrize("is_vector_indexed", ["is_vector_indexed", "not_vector_indexed"])
@pytest.mark.parametrize("segment_status", ["only_growing", "all"])
@pytest.mark.parametrize("index_type", ["HNSW", "BIN_IVF_FLAT"]) #"IVF_FLAT", "HNSW", "BIN_IVF_FLAT"
@pytest.mark.parametrize("index_type", ["HNSW", "BIN_IVF_FLAT"]) # "IVF_FLAT", "HNSW", "BIN_IVF_FLAT"
def test_task_all(self, index_type, is_compacted,
segment_status, is_vector_indexed, is_string_indexed, replica_number, is_deleted, data_size):
segment_status, is_string_indexed, replica_number, is_deleted, data_size):
"""
before reinstall: create collection and insert data, load and search
"""
name = ""
for k,v in locals().items():
for k, v in locals().items():
if k in ["self", "name"]:
continue
name += f"_{k}_{v}"
@ -82,7 +88,7 @@ class TestActionFirstDeployment(TestDeployBase):
pytest.skip("skip test, not enough nodes")
log.info(f"collection name: {name}, replica_number: {replica_number}, is_compacted: {is_compacted},"
f"is_deleted: {is_deleted}, is_vector_indexed: {is_vector_indexed}, is_string_indexed: {is_string_indexed},"
f"is_deleted: {is_deleted}, is_string_indexed: {is_string_indexed},"
f"segment_status: {segment_status}, index_type: {index_type}")
is_binary = True if "BIN" in index_type else False
@ -100,6 +106,22 @@ class TestActionFirstDeployment(TestDeployBase):
# init collection and insert with small size data without flush to get growing segment
collection_w = self.init_collection_general(insert_data=True, is_binary=is_binary, nb=3000,
is_flush=False, is_index=True, name=name)[0]
# params for creating index
if is_binary:
default_index_field = ct.default_binary_vec_field_name
else:
default_index_field = ct.default_float_vec_field_name
# create index for vector
default_index_param = gen_index_param(index_type)
collection_w.create_index(default_index_field, default_index_param)
# create index for string
if is_string_indexed == "is_string_indexed":
default_string_index_params = {}
default_string_index_name = "_default_string_idx"
collection_w.create_index(
default_string_field_name, default_string_index_params, index_name=default_string_index_name)
# load for growing segment
if replica_number >= 1:
try:
@ -107,10 +129,11 @@ class TestActionFirstDeployment(TestDeployBase):
except Exception as e:
log.error(
f"release collection failed: {e} maybe the collection is not loaded")
collection_w.load(replica_number=replica_number)
collection_w.load(replica_number=replica_number, timeout=TIMEOUT)
self.utility_wrap.wait_for_loading_complete(name)
# delete data for growing segment
delete_expr = f"{ct.default_int64_field_name} in {[i for i in range(0,10)]}"
delete_expr = f"{ct.default_int64_field_name} in {[i for i in range(0, 10)]}"
if is_deleted == "is_deleted":
collection_w.delete(expr=delete_expr)
@ -121,10 +144,10 @@ class TestActionFirstDeployment(TestDeployBase):
default_search_exp,
check_task=CheckTasks.check_search_results,
check_items={"nq": default_nq,
"limit": default_limit})
"limit": default_limit})
output_fields = [ct.default_int64_field_name]
collection_w.query(default_term_expr, output_fields=output_fields,
check_task=CheckTasks.check_query_not_empty)
check_task=CheckTasks.check_query_not_empty)
# skip subsequent operations when segment_status is set to only_growing
if segment_status == "only_growing":
@ -135,31 +158,17 @@ class TestActionFirstDeployment(TestDeployBase):
self.init_collection_general(insert_data=True, is_binary=is_binary, nb=data_size,
is_flush=False, is_index=True, name=name)
# at this step, all segment are sealed
collection_w.flush()
if pymilvus_version >= "2.2.0":
collection_w.flush()
else:
collection_w.collection.num_entities
# delete data for sealed segment and before index
delete_expr = f"{ct.default_int64_field_name} in {[i for i in range(10,20)]}"
delete_expr = f"{ct.default_int64_field_name} in {[i for i in range(10, 20)]}"
if is_deleted == "is_deleted":
collection_w.delete(expr=delete_expr)
# params for creating index
if is_binary:
default_index_field = ct.default_binary_vec_field_name
else:
default_index_field = ct.default_float_vec_field_name
collection_w.delete(expr=delete_expr)
# create index for vector
if is_vector_indexed == "is_vector_indexed":
default_index_param = gen_index_param(index_type)
collection_w.create_index(default_index_field, default_index_param)
# create index for string
if is_string_indexed == "is_string_indexed":
default_string_index_params = {}
default_string_index_name = "_default_string_idx"
collection_w.create_index(
default_string_field_name, default_string_index_params, index_name=default_string_index_name)
# delete data for sealed segment and afer index
delete_expr = f"{ct.default_int64_field_name} in {[i for i in range(20,30)]}"
# delete data for sealed segment and after index
delete_expr = f"{ct.default_int64_field_name} in {[i for i in range(20, 30)]}"
if is_deleted == "is_deleted":
collection_w.delete(expr=delete_expr)
if is_compacted == "is_compacted":
@ -171,13 +180,14 @@ class TestActionFirstDeployment(TestDeployBase):
# reload after flush and creating index
if replica_number > 0:
collection_w.release()
collection_w.load(replica_number=replica_number)
collection_w.load(replica_number=replica_number, timeout=TIMEOUT)
self.utility_wrap.wait_for_loading_complete(name)
# insert data to get growing segment after reload
if segment_status == "all":
self.init_collection_general(insert_data=True, is_binary=is_binary, nb=3000,
is_flush=False, is_index=True, name=name)
# search and query for sealed and growing segment
if replica_number > 0:
collection_w.search(vectors_to_search[:default_nq], default_search_field,
@ -185,7 +195,7 @@ class TestActionFirstDeployment(TestDeployBase):
default_search_exp,
check_task=CheckTasks.check_search_results,
check_items={"nq": default_nq,
"limit": default_limit})
"limit": default_limit})
output_fields = [ct.default_int64_field_name]
collection_w.query(default_term_expr, output_fields=output_fields,
check_task=CheckTasks.check_query_not_empty)
check_task=CheckTasks.check_query_not_empty)