From 5bbace10940761e48d3eafd2be2d86fe70df68b9 Mon Sep 17 00:00:00 2001 From: nico <109071306+NicoYuan1986@users.noreply.github.com> Date: Wed, 7 Feb 2024 10:54:47 +0800 Subject: [PATCH] enhance: update test cases and increase timeout of rbac pipeline (#30549) issue: #29880 Signed-off-by: nico --- ci/jenkins/Nightly.groovy | 2 +- .../python_client/base/high_level_api_wrapper.py | 6 +++--- .../milvus_client/test_milvus_client_alias.py | 12 ++++++++---- .../milvus_client/test_milvus_client_index.py | 11 ++++++++--- .../milvus_client/test_milvus_client_search.py | 1 + tests/python_client/requirements.txt | 2 +- tests/python_client/testcases/test_collection.py | 16 ++++++++-------- .../testcases/test_high_level_api.py | 1 + tests/python_client/testcases/test_index.py | 2 +- tests/python_client/testcases/test_insert.py | 2 +- tests/python_client/testcases/test_search.py | 3 ++- 11 files changed, 35 insertions(+), 23 deletions(-) diff --git a/ci/jenkins/Nightly.groovy b/ci/jenkins/Nightly.groovy index ecd7f31f7f..d258b7eaec 100644 --- a/ci/jenkins/Nightly.groovy +++ b/ci/jenkins/Nightly.groovy @@ -208,7 +208,7 @@ pipeline { } else if("${MILVUS_SERVER_TYPE}" == "standalone-authentication") { tag="RBAC" parallel_num = 1 - e2e_timeout_seconds = 1 * 60 * 60 + e2e_timeout_seconds = 3 * 60 * 60 } if ("${MILVUS_CLIENT}" == "pymilvus") { sh """ diff --git a/tests/python_client/base/high_level_api_wrapper.py b/tests/python_client/base/high_level_api_wrapper.py index 0f623d5bbd..b847a02e3b 100644 --- a/tests/python_client/base/high_level_api_wrapper.py +++ b/tests/python_client/base/high_level_api_wrapper.py @@ -515,14 +515,14 @@ class HighLevelApiWrapper: return res, check_result @trace() - def list_aliases(self, client, timeout=None, check_task=None, check_items=None, **kwargs): + def list_aliases(self, client, collection_name, timeout=None, check_task=None, check_items=None, **kwargs): timeout = TIMEOUT if timeout is None else timeout kwargs.update({"timeout": timeout}) func_name = sys._getframe().f_code.co_name - res, check = api_request([client.list_aliases], **kwargs) + res, check = api_request([client.list_aliases, collection_name], **kwargs) check_result = ResponseChecker(res, func_name, check_task, - check_items, check, + check_items, check, collection_name=collection_name, **kwargs).run() return res, check_result diff --git a/tests/python_client/milvus_client/test_milvus_client_alias.py b/tests/python_client/milvus_client/test_milvus_client_alias.py index c1f828fb76..686ecc3ddb 100644 --- a/tests/python_client/milvus_client/test_milvus_client_alias.py +++ b/tests/python_client/milvus_client/test_milvus_client_alias.py @@ -406,7 +406,8 @@ class TestMilvusClientAliasValid(TestcaseBase): # 1. create collection client_w.create_collection(client, collection_name, default_dim, consistency_level="Strong") # 2. create alias - client_w.create_alias(collection_name, alias) + client_w.drop_alias(client, alias) + client_w.create_alias(client, collection_name, alias) collection_name = alias # 2. insert rng = np.random.default_rng(seed=19530) @@ -431,7 +432,10 @@ class TestMilvusClientAliasValid(TestcaseBase): "with_vec": True, "primary_field": default_primary_key_field_name}) client_w.release_collection(client, collection_name) - client_w.drop_collection(client, collection_name) + client_w.drop_collection(client, collection_name, check_task=CheckTasks.err_res, + check_items={ct.err_code: 65535, + ct.err_msg: "cannot drop the collection via alias = collection_alias"}) + client_w.drop_alias(client, alias) @pytest.mark.tags(CaseLabel.L1) @pytest.mark.xfail(reason="pymilvus issue 1891, 1892") @@ -491,10 +495,10 @@ class TestMilvusClientAliasValid(TestcaseBase): client_w.alter_alias(client, collection_name, another_alias) client_w.describe_alias(client, alias) # 3. list alias - aliases = client_w.list_aliases(client)[0] + aliases = client_w.list_aliases(client, collection_name)[0] # assert alias in aliases # assert another_alias in aliases # 4. assert collection is equal to alias according to partitions partition_name_list_alias = client_w.list_partitions(client, another_alias)[0] assert partition_name_list == partition_name_list_alias - client_w.drop_collection(client, collection_name) \ No newline at end of file + client_w.drop_collection(client, collection_name) diff --git a/tests/python_client/milvus_client/test_milvus_client_index.py b/tests/python_client/milvus_client/test_milvus_client_index.py index 86cca7d4c3..89dbf5cf81 100644 --- a/tests/python_client/milvus_client/test_milvus_client_index.py +++ b/tests/python_client/milvus_client/test_milvus_client_index.py @@ -271,6 +271,7 @@ class TestMilvusClientIndexValid(TestcaseBase): """ @pytest.mark.tags(CaseLabel.L1) + @pytest.mark.skip("https://github.com/milvus-io/pymilvus/issues/1886") @pytest.mark.parametrize("index, params", zip(ct.all_index_types[:7], ct.default_index_params[:7])) @@ -290,7 +291,7 @@ class TestMilvusClientIndexValid(TestcaseBase): assert res == [] # 2. prepare index params index_params = client_w.prepare_index_params(client)[0] - index_params.add_index(field_name = "vector", index_type=index, metric_type = metric_type) + index_params.add_index(field_name="vector", index_type=index, metric_type=metric_type) # 3. create index client_w.create_index(client, collection_name, index_params) # 4. create same index twice @@ -319,7 +320,6 @@ class TestMilvusClientIndexValid(TestcaseBase): "primary_field": default_primary_key_field_name}) client_w.drop_collection(client, collection_name) - @pytest.mark.tags(CaseLabel.L2) @pytest.mark.skip(reason="pymilvus issue 1884") @pytest.mark.parametrize("index, params", @@ -369,6 +369,7 @@ class TestMilvusClientIndexValid(TestcaseBase): client_w.drop_collection(client, collection_name) @pytest.mark.tags(CaseLabel.L2) + @pytest.mark.skip("wait for modification") @pytest.mark.parametrize("index, params", zip(ct.all_index_types[:7], ct.default_index_params[:7])) @@ -414,6 +415,7 @@ class TestMilvusClientIndexValid(TestcaseBase): client_w.drop_collection(client, collection_name) @pytest.mark.tags(CaseLabel.L1) + @pytest.mark.skip("wait for modification") def test_milvus_client_index_auto_index(self, scalar_index, metric_type): """ target: test search (high level api) normal case @@ -465,6 +467,7 @@ class TestMilvusClientIndexValid(TestcaseBase): client_w.drop_collection(client, collection_name) @pytest.mark.tags(CaseLabel.L2) + @pytest.mark.skip("wait for modification") def test_milvus_client_index_multiple_vectors(self, scalar_index, metric_type): """ target: test search (high level api) normal case @@ -512,6 +515,7 @@ class TestMilvusClientIndexValid(TestcaseBase): client_w.drop_collection(client, collection_name) @pytest.mark.tags(CaseLabel.L2) + @pytest.mark.skip("wait for modification") @pytest.mark.parametrize("index, params", zip(ct.all_index_types[:7], ct.default_index_params[:7])) @@ -563,6 +567,7 @@ class TestMilvusClientIndexValid(TestcaseBase): client_w.drop_collection(client, collection_name) @pytest.mark.tags(CaseLabel.L2) + @pytest.mark.skip("wait for modification") @pytest.mark.parametrize("index, params", zip(ct.all_index_types[:7], ct.default_index_params[:7])) @@ -582,7 +587,7 @@ class TestMilvusClientIndexValid(TestcaseBase): assert res == [] # 2. prepare index params index_params = client_w.prepare_index_params(client)[0] - index_params.add_index(field_name = "vector", metric_type = metric_type) + index_params.add_index(field_name="vector", metric_type=metric_type) # 3. create index client_w.create_index(client, collection_name, index_params) # 4. drop index diff --git a/tests/python_client/milvus_client/test_milvus_client_search.py b/tests/python_client/milvus_client/test_milvus_client_search.py index 53ee21305b..549d0936e0 100644 --- a/tests/python_client/milvus_client/test_milvus_client_search.py +++ b/tests/python_client/milvus_client/test_milvus_client_search.py @@ -130,6 +130,7 @@ class TestMilvusClientSearchInvalid(TestcaseBase): check_task=CheckTasks.err_res, check_items=error) @pytest.mark.tags(CaseLabel.L2) + @pytest.mark.skip("https://github.com/milvus-io/milvus/issues/29880") def test_milvus_client_search_not_consistent_metric_type(self, metric_type): """ target: test search with inconsistent metric type (default is IP) with that of index diff --git a/tests/python_client/requirements.txt b/tests/python_client/requirements.txt index ec38a0d74e..d01827b97a 100644 --- a/tests/python_client/requirements.txt +++ b/tests/python_client/requirements.txt @@ -12,7 +12,7 @@ allure-pytest==2.7.0 pytest-print==0.2.1 pytest-level==0.1.1 pytest-xdist==2.5.0 -pymilvus==2.4.0rc24 +pymilvus==2.4.0rc33 pytest-rerunfailures==9.1.1 git+https://github.com/Projectplace/pytest-tags ndg-httpsclient diff --git a/tests/python_client/testcases/test_collection.py b/tests/python_client/testcases/test_collection.py index 8e6e751f08..ea32c127b8 100644 --- a/tests/python_client/testcases/test_collection.py +++ b/tests/python_client/testcases/test_collection.py @@ -4225,21 +4225,21 @@ class TestCollectionARRAY(TestcaseBase): {"field_id": 101, "name": "float_vector", "description": "", "type": 101, "params": {"dim": ct.default_dim}, "element_type": 0}, {"field_id": 102, "name": "int8_array", "description": "", "type": 22, - "params": {"max_capacity": "2000"}, "element_type": 2}, + "params": {"max_capacity": 2000}, "element_type": 2}, {"field_id": 103, "name": "int16_array", "description": "", "type": 22, - "params": {"max_capacity": "2000"}, "element_type": 3}, + "params": {"max_capacity": 2000}, "element_type": 3}, {"field_id": 104, "name": "int32_array", "description": "", "type": 22, - "params": {"max_capacity": "2000"}, "element_type": 4}, + "params": {"max_capacity": 2000}, "element_type": 4}, {"field_id": 105, "name": "int64_array", "description": "", "type": 22, - "params": {"max_capacity": "2000"}, "element_type": 5}, + "params": {"max_capacity": 2000}, "element_type": 5}, {"field_id": 106, "name": "bool_array", "description": "", "type": 22, - "params": {"max_capacity": "2000"}, "element_type": 1}, + "params": {"max_capacity": 2000}, "element_type": 1}, {"field_id": 107, "name": "float_array", "description": "", "type": 22, - "params": {"max_capacity": "2000"}, "element_type": 10}, + "params": {"max_capacity": 2000}, "element_type": 10}, {"field_id": 108, "name": "double_array", "description": "", "type": 22, - "params": {"max_capacity": "2000"}, "element_type": 11}, + "params": {"max_capacity": 2000}, "element_type": 11}, {"field_id": 109, "name": "string_array", "description": "", "type": 22, - "params": {"max_length": "100", "max_capacity": "2000"}, "element_type": 21} + "params": {"max_length": 100, "max_capacity": 2000}, "element_type": 21} ] assert res["fields"] == fields diff --git a/tests/python_client/testcases/test_high_level_api.py b/tests/python_client/testcases/test_high_level_api.py index 2e6b1a6652..e321627b5e 100644 --- a/tests/python_client/testcases/test_high_level_api.py +++ b/tests/python_client/testcases/test_high_level_api.py @@ -130,6 +130,7 @@ class TestHighLevelApi(TestcaseBase): check_task=CheckTasks.err_res, check_items=error) @pytest.mark.tags(CaseLabel.L2) + @pytest.mark.skip("https://github.com/milvus-io/milvus/issues/29880") def test_high_level_search_not_consistent_metric_type(self, metric_type): """ target: test search with inconsistent metric type (default is IP) with that of index diff --git a/tests/python_client/testcases/test_index.py b/tests/python_client/testcases/test_index.py index a25ddf7c47..c540abf54d 100644 --- a/tests/python_client/testcases/test_index.py +++ b/tests/python_client/testcases/test_index.py @@ -1788,7 +1788,7 @@ class TestIndexDiskann(TestcaseBase): assert len(collection_w.indexes) == 1 collection_w.release() collection_w.drop_index(index_name=index_name1) - assert collection_w.has_index(index_name=index_name1)[0] == False + assert collection_w.has_index(index_name=index_name1)[0] is False @pytest.mark.tags(CaseLabel.L2) def test_drop_diskann_index_and_create_again(self): diff --git a/tests/python_client/testcases/test_insert.py b/tests/python_client/testcases/test_insert.py index a32d694f5c..c6275a30e5 100644 --- a/tests/python_client/testcases/test_insert.py +++ b/tests/python_client/testcases/test_insert.py @@ -1455,7 +1455,7 @@ class TestInsertInvalid(TestcaseBase): method: insert with nan value: list, number, string ... expected: raise exception """ - if isinstance(json_value, list): + if isinstance(json_value, list) or json_value is None: pytest.skip("invalid in dataframe") collection_name = cf.gen_unique_str(prefix) collection_w = self.init_collection_wrap(name=collection_name) diff --git a/tests/python_client/testcases/test_search.py b/tests/python_client/testcases/test_search.py index 7d86c2038d..d1ec0d359c 100644 --- a/tests/python_client/testcases/test_search.py +++ b/tests/python_client/testcases/test_search.py @@ -9838,7 +9838,8 @@ class TestSearchGroupBy(TestcaseBase): err_code = 999 err_msg = "Unexpected index" if index in ["IVF_FLAT", "IVF_SQ8", "IVF_PQ", "SCANN"]: - err_msg = "not supported for current index type" + err_code = 65535 + err_msg = "Returned knowhere iterator has non-ready iterators inside, terminate group_by operation" collection_w.search(data=search_vectors, anns_field=ct.default_float_vec_field_name, param=search_params, limit=limit, group_by_field=ct.default_int8_field_name,