diff --git a/tests/python_client/base/client_v2_base.py b/tests/python_client/base/client_v2_base.py index 20b67c26b5..dbebed2ce2 100644 --- a/tests/python_client/base/client_v2_base.py +++ b/tests/python_client/base/client_v2_base.py @@ -237,12 +237,12 @@ class TestMilvusClientV2Base(Base): return res, check_result @trace() - def delete(self, client, collection_name, timeout=None, check_task=None, check_items=None, **kwargs): + def delete(self, client, collection_name, ids=None, timeout=None, filter=None, partition_name=None, + check_task=None, check_items=None, **kwargs): timeout = TIMEOUT if timeout is None else timeout - kwargs.update({"timeout": timeout}) func_name = sys._getframe().f_code.co_name - res, check = api_request([client.delete, collection_name], **kwargs) + res, check = api_request([client.delete, collection_name, ids, timeout, filter, partition_name], **kwargs) check_result = ResponseChecker(res, func_name, check_task, check_items, check, collection_name=collection_name, **kwargs).run() diff --git a/tests/python_client/check/func_check.py b/tests/python_client/check/func_check.py index 970fddeede..8759632b86 100644 --- a/tests/python_client/check/func_check.py +++ b/tests/python_client/check/func_check.py @@ -469,16 +469,20 @@ class ResponseChecker: if func_name != 'search_iterator': log.warning("The function name is {} rather than {}".format(func_name, "search_iterator")) search_iterator = search_res + expected_batch_size = check_items.get("batch_size", None) + expected_iterate_times = check_items.get("iterate_times", None) pk_list = [] + iterate_times = 0 while True: try: res = search_iterator.next() + iterate_times += 1 if not res: log.info("search iteration finished, close") search_iterator.close() break - if check_items.get("batch_size", None): - assert len(res) <= check_items["batch_size"] + if expected_batch_size is not None: + assert len(res) <= expected_batch_size if check_items.get("radius", None): for distance in res.distances(): if check_items["metric_type"] == "L2": @@ -495,12 +499,14 @@ class ResponseChecker: except Exception as e: assert check_items["err_msg"] in str(e) return False - expected_batch_size = check_items.get("batch_size", None) - if expected_batch_size is not None and expected_batch_size == 0: # expected batch size =0 if external filter all - assert len(pk_list) == 0 - assert len(pk_list) == len(set(pk_list)) - log.info("check: total %d results" % len(pk_list)) - + if expected_iterate_times is not None: + assert iterate_times <= expected_iterate_times + if expected_iterate_times == 1: + assert len(pk_list) == 0 # expected batch size =0 if external filter all + assert iterate_times == 1 + return True + log.debug(f"check: total {len(pk_list)} results, set len: {len(set(pk_list))}, iterate_times: {iterate_times}") + assert len(pk_list) == len(set(pk_list)) != 0 return True @staticmethod diff --git a/tests/python_client/milvus_client/test_milvus_client_alias.py b/tests/python_client/milvus_client/test_milvus_client_alias.py index 6fce497f8c..cffe3ea42c 100644 --- a/tests/python_client/milvus_client/test_milvus_client_alias.py +++ b/tests/python_client/milvus_client/test_milvus_client_alias.py @@ -147,14 +147,23 @@ class TestMilvusClientAliasInvalid(TestMilvusClientV2Base): expected: create alias successfully """ client = self._client() - collection_name = cf.gen_unique_str(prefix) + collection_name = cf.gen_unique_str('coll') # 1. create collection self.create_collection(client, collection_name, default_dim, consistency_level="Strong") # 2. create alias - error = {ct.err_code: 1601, ct.err_msg: f"alias and collection name conflict[database=default]" - f"[alias={collection_name}]"} + error = {ct.err_code: 1601, + ct.err_msg: f"alias and collection name conflict[database=default][alias={collection_name}]"} self.create_alias(client, collection_name, collection_name, check_task=CheckTasks.err_res, check_items=error) + # create a collection with the same alias name + alias_name = cf.gen_unique_str('alias') + self.create_alias(client, collection_name, alias_name) + error = {ct.err_code: 1601, + ct.err_msg: f"collection name [{alias_name}] conflicts with an existing alias, " + f"please choose a unique name"} + self.create_collection(client, alias_name, default_dim, + check_task=CheckTasks.err_res, check_items=error) + self.drop_alias(client, alias_name) self.drop_collection(client, collection_name) @pytest.mark.tags(CaseLabel.L1) diff --git a/tests/python_client/milvus_client/test_milvus_client_search_interator.py b/tests/python_client/milvus_client/test_milvus_client_search_interator.py index dcdd0e8541..8bbeb9f190 100644 --- a/tests/python_client/milvus_client/test_milvus_client_search_interator.py +++ b/tests/python_client/milvus_client/test_milvus_client_search_interator.py @@ -46,15 +46,24 @@ def external_filter_nothing(hits): return hits +def external_filter_invalid_arguments(hits, iaminvalid): + pass + + +def external_filter_with_outputs(hits): + results = [] + for hit in hits: + # equals filter nothing if there are output_fields + if hit.distance < 1.0 and len(hit.fields) > 0: + results.append(hit) + return results + + class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): """ Test case of search iterator interface """ - @pytest.fixture(scope="function", params=[{}, {"radius": 0.1, "range_filter": 0.9}]) - def search_params(self, request): - yield request.param - @pytest.mark.tags(CaseLabel.L1) - def test_milvus_client_search_iterator_using_mul_db(self, search_params): + def test_milvus_client_search_iterator_using_mul_db(self): """ target: test search iterator(high level api) case about mul db method: create connection, collection, insert and search iterator @@ -71,8 +80,7 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): collections = self.list_collections(client)[0] assert collection_name in collections # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) @@ -85,8 +93,8 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): self.insert(client, collection_name, rows) self.flush(client, collection_name) # 5. search_iterator - vectors_to_search = rng.random((1, default_dim)) - search_params = {"params": search_params} + vectors_to_search = cf.gen_vectors(1, default_dim) + search_params = {"params": {}} error_msg = "alias or database may have been changed" self.search_iterator(client, collection_name, vectors_to_search, batch_size, search_params=search_params, use_mul_db=True, another_db=my_db, @@ -96,7 +104,7 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): self.drop_collection(client, collection_name) @pytest.mark.tags(CaseLabel.L1) - def test_milvus_client_search_iterator_alias_different_col(self, search_params): + def test_milvus_client_search_iterator_alias_different_col(self): """ target: test search iterator(high level api) case about alias method: create connection, collection, insert and search iterator @@ -117,16 +125,15 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): collections = self.list_collections(client)[0] assert collection_name_new in collections # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) self.insert(client, collection_name_new, rows) self.flush(client, collection_name_new) # 3. search_iterator - vectors_to_search = rng.random((1, default_dim)) - search_params = {"params": search_params} + vectors_to_search = cf.gen_vectors(1, default_dim) + search_params = {"params": {}} error_msg = "alias or database may have been changed" self.search_iterator(client, alias, vectors_to_search, batch_size, search_params=search_params, use_alias=True, another_collection=collection_name_new, @@ -151,8 +158,7 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): error = {ct.err_code: 100, ct.err_msg: f"collection not found[database=default]" f"[collection={collection_name}]"} - rng = np.random.default_rng(seed=19530) - vectors_to_search = rng.random((1, default_dim)) + vectors_to_search = cf.gen_vectors(1, default_dim) insert_ids = [i for i in range(default_nb)] self.search_iterator(client, collection_name, vectors_to_search, batch_size=5, @@ -180,8 +186,7 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): "dim": default_dim, "consistency_level": 2}) # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) @@ -216,8 +221,7 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): "dim": default_dim, "consistency_level": 2}) # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) @@ -255,13 +259,12 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): "dim": default_dim, "consistency_level": 2}) # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) # 3. search - vectors_to_search = rng.random((1, default_dim)) + vectors_to_search = cf.gen_vectors(1, default_dim) error = {ct.err_code: 1, ct.err_msg: f"batch size cannot be less than zero"} self.search_iterator(client, collection_name, vectors_to_search, @@ -292,13 +295,12 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): "dim": default_dim, "consistency_level": 2}) # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) # 3. search - vectors_to_search = rng.random((1, default_dim)) + vectors_to_search = cf.gen_vectors(1, default_dim) error = {ct.err_code: 1100, ct.err_msg: f"failed to create query plan: predicate is not a boolean expression: invalidexpr, " f"data type: JSON: invalid parameter"} @@ -310,7 +312,7 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): self.release_collection(client, collection_name) self.drop_collection(client, collection_name) - @pytest.mark.tags(CaseLabel.L1) + @pytest.mark.tags(CaseLabel.L2) @pytest.mark.parametrize("limit", [-10]) @pytest.mark.skip("https://github.com/milvus-io/milvus/issues/39066") def test_milvus_client_search_iterator_with_invalid_limit(self, limit): @@ -333,13 +335,12 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): "dim": default_dim, "consistency_level": 2}) # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) # 3. search - vectors_to_search = rng.random((1, default_dim)) + vectors_to_search = cf.gen_vectors(1, default_dim) error = {ct.err_code: 1, ct.err_msg: f"`limit` value {limit} is illegal"} self.search_iterator(client, collection_name, vectors_to_search, @@ -350,7 +351,7 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): self.release_collection(client, collection_name) self.drop_collection(client, collection_name) - @pytest.mark.tags(CaseLabel.L1) + @pytest.mark.tags(CaseLabel.L2) @pytest.mark.parametrize("output_fields", ["id"]) @pytest.mark.skip("A field that does not currently exist will simply have no effect, " "but it would be better if an error were reported.") @@ -374,14 +375,13 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): "dim": default_dim, "consistency_level": 2}) # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) # 3. search - vectors_to_search = rng.random((1, default_dim)) + vectors_to_search = cf.gen_vectors(1, default_dim) error = {ct.err_code: 1, ct.err_msg: f"`output_fields` value {output_fields} is illegal"} self.search_iterator(client, collection_name, vectors_to_search, @@ -417,14 +417,13 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): "dim": default_dim, "consistency_level": 2}) # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) # 3. search - vectors_to_search = rng.random((1, default_dim)) + vectors_to_search = cf.gen_vectors(1, default_dim) error = {ct.err_code: 1, ct.err_msg: f"'str' object has no attribute 'get'"} self.search_iterator(client, collection_name, vectors_to_search, @@ -437,7 +436,7 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): self.release_collection(client, collection_name) self.drop_collection(client, collection_name) - @pytest.mark.tags(CaseLabel.L1) + @pytest.mark.tags(CaseLabel.L2) @pytest.mark.parametrize("partition_name", ["client_partition_85Jv3Pf3"]) def test_milvus_client_search_iterator_with_invalid_partition_name(self, partition_name): """ @@ -460,14 +459,13 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): "consistency_level": 2, "num_partitions": 2}) # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) # 3. search - vectors_to_search = rng.random((1, default_dim)) + vectors_to_search = cf.gen_vectors(1, default_dim) error = {ct.err_code: 1, ct.err_msg: f"`partition_name_array` value {partition_name} is illegal"} self.search_iterator(client, collection_name, vectors_to_search, @@ -480,7 +478,7 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): self.release_collection(client, collection_name) self.drop_collection(client, collection_name) - @pytest.mark.tags(CaseLabel.L1) + @pytest.mark.tags(CaseLabel.L2) @pytest.mark.parametrize("partition_name", ["nonexistent"]) def test_milvus_client_search_iterator_with_nonexistent_partition_name(self, partition_name): """ @@ -501,14 +499,13 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): "dim": default_dim, "consistency_level": 2}) # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) # 3. search - vectors_to_search = rng.random((1, default_dim)) + vectors_to_search = cf.gen_vectors(1, default_dim) error = {ct.err_code: 65535, ct.err_msg: f"partition name {partition_name} not found"} self.search_iterator(client, collection_name, vectors_to_search, @@ -521,7 +518,7 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): self.release_collection(client, collection_name) self.drop_collection(client, collection_name) - @pytest.mark.tags(CaseLabel.L1) + @pytest.mark.tags(CaseLabel.L2) @pytest.mark.parametrize("anns_field", ["nonexistent", ]) def test_milvus_client_search_iterator_with_nonexistent_anns_field(self, anns_field): """ @@ -542,14 +539,13 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): "dim": default_dim, "consistency_level": 2}) # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) # 3. search - vectors_to_search = rng.random((1, default_dim)) + vectors_to_search = cf.gen_vectors(1, default_dim) error = {ct.err_code: 1100, ct.err_msg: f"failed to create query plan: failed to get field schema by name: " f"fieldName({anns_field}) not found: invalid parameter"} @@ -563,7 +559,7 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): self.release_collection(client, collection_name) self.drop_collection(client, collection_name) - @pytest.mark.tags(CaseLabel.L1) + @pytest.mark.tags(CaseLabel.L2) @pytest.mark.parametrize("round_decimal", ["tt"]) def test_milvus_client_search_iterator_with_invalid_round_decimal(self, round_decimal): """ @@ -584,14 +580,13 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): "dim": default_dim, "consistency_level": 2}) # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) # 3. search - vectors_to_search = rng.random((1, default_dim)) + vectors_to_search = cf.gen_vectors(1, default_dim) error = {ct.err_code: 1, ct.err_msg: f"`round_decimal` value {round_decimal} is illegal"} self.search_iterator(client, collection_name, vectors_to_search, @@ -604,6 +599,45 @@ class TestMilvusClientSearchIteratorInValid(TestMilvusClientV2Base): self.release_collection(client, collection_name) self.drop_collection(client, collection_name) + @pytest.mark.tags(CaseLabel.L2) + def test_milvus_client_search_iterator_with_invalid_external_func(self): + """ + target: test search iterator (high level api) normal case + method: create connection, collection, insert and search iterator + expected: search iterator successfully + """ + batch_size = 20 + client = self._client() + collection_name = cf.gen_unique_str(prefix) + self.using_database(client, "default") + # 1. create collection + self.create_collection(client, collection_name, default_dim, consistency_level="Bounded") + collections = self.list_collections(client)[0] + assert collection_name in collections + self.describe_collection(client, collection_name, + check_task=CheckTasks.check_describe_collection_property, + check_items={"collection_name": collection_name, + "dim": default_dim}) + # 2. insert + rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), + default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] + self.insert(client, collection_name, rows) + self.flush(client, collection_name) + # 3. search iterator + vectors_to_search = cf.gen_vectors(1, default_dim) + search_params = {} + with pytest.raises(TypeError, match="got an unexpected keyword argument 'metric_type'"): + self.search_iterator(client, collection_name, vectors_to_search, batch_size, + search_params=search_params, limit=100, + external_filter_func=external_filter_invalid_arguments(metric_type="L2"), + check_task=CheckTasks.check_nothing) + it = self.search_iterator(client, collection_name, vectors_to_search, batch_size, + search_params=search_params, limit=100, + external_filter_func=external_filter_invalid_arguments, + check_task=CheckTasks.check_nothing)[0] + with pytest.raises(TypeError, match="missing 1 required positional argument: 'iaminvalid'"): + it.next() + class TestMilvusClientSearchIteratorValid(TestMilvusClientV2Base): """ Test case of search iterator interface """ @@ -616,10 +650,6 @@ class TestMilvusClientSearchIteratorValid(TestMilvusClientV2Base): def metric_type(self, request): yield request.param - @pytest.fixture(scope="function", params=[{}, {"radius": 0.1, "range_filter": 0.9}]) - def search_params(self, request): - yield request.param - """ ****************************************************************** # The following are valid base cases @@ -627,7 +657,8 @@ class TestMilvusClientSearchIteratorValid(TestMilvusClientV2Base): """ @pytest.mark.tags(CaseLabel.L0) - def test_milvus_client_search_iterator_default(self, search_params): + @pytest.mark.parametrize("metric_type", ct.float_metrics) + def test_milvus_client_search_iterator_default(self, metric_type): """ target: test search iterator (high level api) normal case method: create connection, collection, insert and search iterator @@ -635,10 +666,12 @@ class TestMilvusClientSearchIteratorValid(TestMilvusClientV2Base): """ batch_size = 20 client = self._client() + collection_name = cf.gen_unique_str(prefix) self.using_database(client, "default") # 1. create collection - self.create_collection(client, collection_name, default_dim, consistency_level="Bounded") + self.create_collection(client, collection_name, default_dim, metric_type=metric_type, + consistency_level="Bounded") collections = self.list_collections(client)[0] assert collection_name in collections self.describe_collection(client, collection_name, @@ -647,299 +680,157 @@ class TestMilvusClientSearchIteratorValid(TestMilvusClientV2Base): "dim": default_dim, "consistency_level": 0}) # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), - default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] + rows = [{default_primary_key_field_name: i, + default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), + default_float_field_name: i * 1.0, + default_string_field_name: str(i)} for i in range(default_nb)] self.insert(client, collection_name, rows) self.flush(client, collection_name) # 3. search iterator - vectors_to_search = rng.random((1, default_dim)) - search_params = {"params": search_params} + vectors_to_search = cf.gen_vectors(1, default_dim) + search_params = {"params": {}} + self.search_iterator(client, collection_name=collection_name, data=vectors_to_search, + anns_field=default_vector_field_name, + search_params=search_params, batch_size=batch_size, + check_task=CheckTasks.check_search_iterator, + check_items={"metric_type": metric_type, "batch_size": batch_size}) + limit = 200 + res = self.search(client, collection_name, vectors_to_search, + search_params=search_params, limit=200, + check_task=CheckTasks.check_search_results, + check_items={"nq": 1, "limit": limit, "enable_milvus_client_api": True})[0] for limit in [batch_size - 3, batch_size, batch_size * 2, -1]: - log.debug(f"search iterator with limit={limit}") + if metric_type != "L2": + radius = res[0][limit // 2].get('distance', 0) - 0.1 # pick a radius to make sure there exists results + range_filter = res[0][0].get('distance', 0) + 0.1 + else: + radius = res[0][limit // 2].get('distance', 0) + 0.1 + range_filter = res[0][0].get('distance', 0) - 0.1 + search_params = {"params": {"radius": radius, "range_filter": range_filter}} + log.debug(f"search iterator with limit={limit} radius={radius}, range_filter={range_filter}") + expected_batch_size = batch_size if limit == -1 else min(batch_size, limit) # external filter not set self.search_iterator(client, collection_name, vectors_to_search, batch_size, search_params=search_params, limit=limit, check_task=CheckTasks.check_search_iterator, - check_items={"batch_size": batch_size if limit == -1 else min(batch_size, limit)}) + check_items={"batch_size": expected_batch_size}) # external filter half self.search_iterator(client, collection_name, vectors_to_search, batch_size, search_params=search_params, limit=limit, external_filter_func=external_filter_half, check_task=CheckTasks.check_search_iterator, - check_items={"batch_size": batch_size if limit == -1 else min(batch_size, limit)}) + check_items={"batch_size": expected_batch_size}) # external filter nothing self.search_iterator(client, collection_name, vectors_to_search, batch_size, search_params=search_params, limit=limit, external_filter_func=external_filter_nothing, check_task=CheckTasks.check_search_iterator, - check_items={"batch_size": batch_size if limit == -1 else min(batch_size, limit)}) + check_items={"batch_size": expected_batch_size}) + # external filter with outputs + self.search_iterator(client, collection_name, vectors_to_search, batch_size, + search_params=search_params, limit=limit, output_fields=["*"], + external_filter_func=external_filter_with_outputs, + check_task=CheckTasks.check_search_iterator, + check_items={"batch_size": expected_batch_size}) # external filter all self.search_iterator(client, collection_name, vectors_to_search, batch_size, search_params=search_params, limit=limit, external_filter_func=external_filter_all, check_task=CheckTasks.check_search_iterator, - check_items={"batch_size": 0}) + check_items={"batch_size": 0, "iterate_times": 1}) self.release_collection(client, collection_name) self.drop_collection(client, collection_name) - @pytest.mark.tags(CaseLabel.L1) - @pytest.mark.parametrize("nullable", [True, False]) - def test_milvus_client_search_iterator_about_nullable_default(self, nullable, search_params): - """ - target: test search iterator (high level api) normal case about nullable and default value - method: create connection, collection, insert and search iterator - expected: search iterator successfully - """ - batch_size = 20 - client = self._client() - collection_name = cf.gen_unique_str(prefix) - dim = 128 - # 1. create collection - schema = self.create_schema(client, enable_dynamic_field=False)[0] - schema.add_field(default_primary_key_field_name, DataType.VARCHAR, max_length=64, is_primary=True, - auto_id=False) - schema.add_field(default_vector_field_name, DataType.FLOAT_VECTOR, dim=dim) - schema.add_field(default_string_field_name, DataType.VARCHAR, max_length=64, is_partition_key=True) - schema.add_field("nullable_field", DataType.INT64, nullable=True, default_value=10) - schema.add_field("array_field", DataType.ARRAY, element_type=DataType.INT64, max_capacity=12, - max_length=64, nullable=True) - index_params = self.prepare_index_params(client)[0] - index_params.add_index(default_vector_field_name, metric_type="COSINE") - self.create_collection(client, collection_name, dimension=dim, schema=schema, index_params=index_params) - # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [ - {default_primary_key_field_name: str(i), default_vector_field_name: list(rng.random((1, default_dim))[0]), - default_string_field_name: str(i), "nullable_field": None, "array_field": None} for i in range(default_nb)] - self.insert(client, collection_name, rows) - self.flush(client, collection_name) - # 3. search iterator - vectors_to_search = rng.random((1, default_dim)) - insert_ids = [i for i in range(default_nb)] - search_params = {"params": search_params} - self.search_iterator(client, collection_name, vectors_to_search, batch_size, filter="nullable_field>=10", - search_params=search_params, - check_task=CheckTasks.check_search_iterator, - check_items={"enable_milvus_client_api": True, - "nq": len(vectors_to_search), - "ids": insert_ids, - "limit": default_limit}) - if self.has_collection(client, collection_name)[0]: - self.drop_collection(client, collection_name) + # @pytest.mark.tags(CaseLabel.L1) + # @pytest.mark.parametrize("nullable", [True, False]) + # @pytest.mark.skip("TODO: need update the case steps and assertion") + # def test_milvus_client_search_iterator_about_nullable_default(self, nullable, search_params): + # """ + # target: test search iterator (high level api) normal case about nullable and default value + # method: create connection, collection, insert and search iterator + # expected: search iterator successfully + # """ + # batch_size = 20 + # client = self._client() + # collection_name = cf.gen_unique_str(prefix) + # dim = 128 + # # 1. create collection + # schema = self.create_schema(client, enable_dynamic_field=False)[0] + # schema.add_field(default_primary_key_field_name, DataType.VARCHAR, max_length=64, is_primary=True, + # auto_id=False) + # schema.add_field(default_vector_field_name, DataType.FLOAT_VECTOR, dim=dim) + # schema.add_field(default_string_field_name, DataType.VARCHAR, max_length=64, is_partition_key=True) + # schema.add_field("nullable_field", DataType.INT64, nullable=True, default_value=10) + # schema.add_field("array_field", DataType.ARRAY, element_type=DataType.INT64, max_capacity=12, + # max_length=64, nullable=True) + # index_params = self.prepare_index_params(client)[0] + # index_params.add_index(default_vector_field_name, metric_type="COSINE") + # self.create_collection(client, collection_name, dimension=dim, schema=schema, index_params=index_params) + # # 2. insert + # rows = [ + # {default_primary_key_field_name: str(i), default_vector_field_name: list(cf.gen_vectors(1, dim)[0]), + # default_string_field_name: str(i), "nullable_field": None, "array_field": None} for i in range(default_nb)] + # self.insert(client, collection_name, rows) + # self.flush(client, collection_name) + # # 3. search iterator + # vectors_to_search = cf.gen_vectors(1, dim) + # insert_ids = [i for i in range(default_nb)] + # search_params = {"params": search_params} + # self.search_iterator(client, collection_name, vectors_to_search, batch_size, filter="nullable_field>=10", + # search_params=search_params, + # check_task=CheckTasks.check_search_iterator, + # check_items={"enable_milvus_client_api": True, + # "nq": len(vectors_to_search), + # "ids": insert_ids, + # "limit": default_limit}) + # if self.has_collection(client, collection_name)[0]: + # self.drop_collection(client, collection_name) + # + # @pytest.mark.tags(CaseLabel.L1) + # @pytest.mark.skip("TODO: need update the case steps and assertion") + # def test_milvus_client_rename_search_iterator_default(self, search_params): + # """ + # target: test search iterator(high level api) normal case + # method: create connection, collection, insert and search iterator + # expected: search iterator successfully + # """ + # batch_size = 20 + # client = self._client() + # collection_name = cf.gen_unique_str(prefix) + # # 1. create collection + # self.create_collection(client, collection_name, default_dim, consistency_level="Bounded") + # collections = self.list_collections(client)[0] + # assert collection_name in collections + # self.describe_collection(client, collection_name, + # check_task=CheckTasks.check_describe_collection_property, + # check_items={"collection_name": collection_name, + # "dim": default_dim, + # "consistency_level": 0}) + # old_name = collection_name + # new_name = collection_name + "new" + # self.rename_collection(client, old_name, new_name) + # # 2. insert + # rows = [{default_primary_key_field_name: i, default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), + # default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] + # self.insert(client, new_name, rows) + # self.flush(client, new_name) + # # assert self.num_entities(client, collection_name)[0] == default_nb + # # 3. search_iterator + # vectors_to_search = cf.gen_vectors(1, default_dim) + # insert_ids = [i for i in range(default_nb)] + # search_params = {"params": search_params} + # self.search_iterator(client, new_name, vectors_to_search, batch_size, search_params=search_params, + # check_task=CheckTasks.check_search_iterator, + # check_items={"enable_milvus_client_api": True, + # "nq": len(vectors_to_search), + # "ids": insert_ids, + # "limit": default_limit}) + # self.release_collection(client, new_name) + # self.drop_collection(client, new_name) @pytest.mark.tags(CaseLabel.L1) - def test_milvus_client_rename_search_iterator_default(self, search_params): - """ - target: test search iterator(high level api) normal case - method: create connection, collection, insert and search iterator - expected: search iterator successfully - """ - batch_size = 20 - client = self._client() - collection_name = cf.gen_unique_str(prefix) - # 1. create collection - self.create_collection(client, collection_name, default_dim, consistency_level="Bounded") - collections = self.list_collections(client)[0] - assert collection_name in collections - self.describe_collection(client, collection_name, - check_task=CheckTasks.check_describe_collection_property, - check_items={"collection_name": collection_name, - "dim": default_dim, - "consistency_level": 0}) - old_name = collection_name - new_name = collection_name + "new" - self.rename_collection(client, old_name, new_name) - # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), - default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] - self.insert(client, new_name, rows) - self.flush(client, new_name) - # assert self.num_entities(client, collection_name)[0] == default_nb - # 3. search_iterator - vectors_to_search = rng.random((1, default_dim)) - insert_ids = [i for i in range(default_nb)] - search_params = {"params": search_params} - self.search_iterator(client, new_name, vectors_to_search, batch_size, search_params=search_params, - check_task=CheckTasks.check_search_iterator, - check_items={"enable_milvus_client_api": True, - "nq": len(vectors_to_search), - "ids": insert_ids, - "limit": default_limit}) - self.release_collection(client, new_name) - self.drop_collection(client, new_name) - - @pytest.mark.tags(CaseLabel.L1) - def test_milvus_client_array_insert_search_iterator(self, search_params): - """ - target: test search iterator (high level api) normal case - method: create connection, collection, insert and search iterator - expected: search iterator successfully - """ - batch_size = 20 - client = self._client() - collection_name = cf.gen_unique_str(prefix) - # 1. create collection - self.create_collection(client, collection_name, default_dim, consistency_level="Strong") - collections = self.list_collections(client)[0] - assert collection_name in collections - # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{ - default_primary_key_field_name: i, - default_vector_field_name: list(rng.random((1, default_dim))[0]), - default_float_field_name: i * 1.0, - default_int32_array_field_name: [i, i + 1, i + 2], - default_string_array_field_name: [str(i), str(i + 1), str(i + 2)] - } for i in range(default_nb)] - self.insert(client, collection_name, rows) - # 3. search iterator - vectors_to_search = rng.random((1, default_dim)) - insert_ids = [i for i in range(default_nb)] - search_params = {"params": search_params} - self.search_iterator(client, collection_name, vectors_to_search, batch_size, search_params=search_params, - check_task=CheckTasks.check_search_iterator, - check_items={"enable_milvus_client_api": True, - "nq": len(vectors_to_search), - "ids": insert_ids, - "limit": default_limit}) - - @pytest.mark.tags(CaseLabel.L2) - def test_milvus_client_search_iterator_string(self, search_params): - """ - target: test search iterator (high level api) for string primary key - method: create connection, collection, insert and search iterator - expected: search iterator successfully - """ - batch_size = 20 - client = self._client() - collection_name = cf.gen_unique_str(prefix) - # 1. create collection - self.create_collection(client, collection_name, default_dim, id_type="string", max_length=ct.default_length) - # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [ - {default_primary_key_field_name: str(i), default_vector_field_name: list(rng.random((1, default_dim))[0]), - default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] - self.insert(client, collection_name, rows) - self.flush(client, collection_name) - # 3. search_iterator - vectors_to_search = rng.random((1, default_dim)) - search_params = {"params": search_params} - self.search_iterator(client, collection_name, vectors_to_search, batch_size, search_params=search_params, - check_task=CheckTasks.check_search_iterator, - check_items={"enable_milvus_client_api": True, - "nq": len(vectors_to_search), - "limit": default_limit}) - self.drop_collection(client, collection_name) - - @pytest.mark.tags(CaseLabel.L2) - def test_milvus_client_search_iterator_different_metric_type_no_specify_in_search_params(self, metric_type, auto_id, - search_params): - """ - target: test search (high level api) normal case - method: create connection, collection, insert and search - expected: search successfully with limit(topK) - """ - client = self._client() - collection_name = cf.gen_unique_str(prefix) - # 1. create collection - self.create_collection(client, collection_name, default_dim, metric_type=metric_type, auto_id=auto_id, - consistency_level="Strong") - # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), - default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] - if auto_id: - for row in rows: - row.pop(default_primary_key_field_name) - self.insert(client, collection_name, rows) - # 3. search_iterator - vectors_to_search = rng.random((1, default_dim)) - search_params = {"params": search_params} - self.search_iterator(client, collection_name, vectors_to_search, batch_size=default_batch_size, - limit=default_limit, search_params=search_params, - output_fields=[default_primary_key_field_name], - check_task=CheckTasks.check_search_iterator, - check_items={"enable_milvus_client_api": True, - "nq": len(vectors_to_search), - "limit": default_limit}) - self.drop_collection(client, collection_name) - - @pytest.mark.tags(CaseLabel.L2) - def test_milvus_client_search_iterator_different_metric_type_specify_in_search_params(self, metric_type, auto_id, - search_params): - """ - target: test search iterator (high level api) normal case - method: create connection, collection, insert and search iterator - expected: search iterator successfully with limit(topK) - """ - client = self._client() - collection_name = cf.gen_unique_str(prefix) - # 1. create collection - self.create_collection(client, collection_name, default_dim, metric_type=metric_type, auto_id=auto_id, - consistency_level="Strong") - # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), - default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] - if auto_id: - for row in rows: - row.pop(default_primary_key_field_name) - self.insert(client, collection_name, rows) - # 3. search_iterator - vectors_to_search = rng.random((1, default_dim)) - search_params = {"params": search_params} - search_params.update({"metric_type": metric_type}) - self.search_iterator(client, collection_name, vectors_to_search, batch_size=default_batch_size, - limit=default_limit, search_params=search_params, - output_fields=[default_primary_key_field_name], - check_task=CheckTasks.check_search_iterator, - check_items={"enable_milvus_client_api": True, - "nq": len(vectors_to_search), - "limit": default_limit}) - self.drop_collection(client, collection_name) - - @pytest.mark.tags(CaseLabel.L1) - def test_milvus_client_search_iterator_delete_with_ids(self, search_params): - """ - target: test delete (high level api) - method: create connection, collection, insert delete, and search iterator - expected: search iterator successfully without deleted data - """ - client = self._client() - collection_name = cf.gen_unique_str(prefix) - # 1. create collection - self.create_collection(client, collection_name, default_dim, consistency_level="Strong") - # 2. insert - default_nb = 1000 - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), - default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] - pks = self.insert(client, collection_name, rows)[0] - # 3. delete - delete_num = 3 - self.delete(client, collection_name, ids=[i for i in range(delete_num)]) - # 4. search_iterator - vectors_to_search = rng.random((1, default_dim)) - insert_ids = [i for i in range(default_nb)] - for insert_id in range(delete_num): - if insert_id in insert_ids: - insert_ids.remove(insert_id) - limit = default_nb - delete_num - search_params = {"params": search_params} - self.search_iterator(client, collection_name, vectors_to_search, batch_size=default_batch_size, - search_params=search_params, limit=default_nb, - check_task=CheckTasks.check_search_iterator, - check_items={"enable_milvus_client_api": True, - "nq": len(vectors_to_search), - "ids": insert_ids, - "limit": limit}) - self.drop_collection(client, collection_name) - - @pytest.mark.tags(CaseLabel.L1) - def test_milvus_client_search_iterator_delete_with_filters(self, search_params): + @pytest.mark.parametrize('id_type', ["int", "string"]) + def test_milvus_client_search_iterator_delete_with_ids(self, id_type): """ target: test delete (high level api) method: create connection, collection, insert delete, and search iterator @@ -948,81 +839,45 @@ class TestMilvusClientSearchIteratorValid(TestMilvusClientV2Base): client = self._client() collection_name = cf.gen_unique_str(prefix) # 1. create collection - self.create_collection(client, collection_name, default_dim, consistency_level="Strong") + self.create_collection(client, collection_name, default_dim, id_type=id_type, max_length=128, + consistency_level="Strong") # 2. insert - default_nb = 1000 - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), + default_nb = 2000 + if id_type == 'int': + rows = [{default_primary_key_field_name: i, + default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), + default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] + else: + rows = [ + {default_primary_key_field_name: cf.gen_unique_str()+str(i), + default_vector_field_name: list(cf.gen_vectors(1, default_dim)[0]), default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] - pks = self.insert(client, collection_name, rows)[0] - # 3. delete - delete_num = 3 - self.delete(client, collection_name, filter=f"id < {delete_num}") - # 4. search_iterator - vectors_to_search = rng.random((1, default_dim)) - insert_ids = [i for i in range(default_nb)] - for insert_id in range(delete_num): - if insert_id in insert_ids: - insert_ids.remove(insert_id) - limit = default_nb - delete_num - search_params = {"params": search_params} - self.search_iterator(client, collection_name, vectors_to_search, batch_size=default_batch_size, - search_params=search_params, limit=default_nb, + self.insert(client, collection_name, rows)[0] + # 3. search_iterator and delete + vectors_to_search = cf.gen_vectors(1, default_dim) + batch_size = 200 + search_params = {"params": {}} + it = self.search_iterator(client, collection_name, vectors_to_search, batch_size=batch_size, + search_params=search_params, limit=500, + check_task=CheckTasks.check_nothing)[0] + res = it.next() + it.close() + delete_ids = res.ids() + self.delete(client, collection_name, ids=delete_ids) + # search iterator again + it2 = self.search_iterator(client, collection_name, vectors_to_search, batch_size=batch_size, + search_params=search_params, limit=500, + check_task=CheckTasks.check_nothing)[0] + res2 = it2.next() + it2.close() + for del_id in delete_ids: + assert del_id not in res2.ids() + # search iterator again + self.search_iterator(client, collection_name, vectors_to_search, batch_size=batch_size, + search_params=search_params, limit=500, check_task=CheckTasks.check_search_iterator, - check_items={"enable_milvus_client_api": True, - "nq": len(vectors_to_search), - "ids": insert_ids, - "limit": limit}) - # 5. query - self.query(client, collection_name, filter=default_search_exp, - check_task=CheckTasks.check_query_results, - check_items={exp_res: rows[delete_num:], - "with_vec": True, - "primary_field": default_primary_key_field_name}) - self.drop_collection(client, collection_name) - - @pytest.mark.tags(CaseLabel.L0) - @pytest.mark.parametrize("metric_type", ["L2"]) - @pytest.mark.parametrize("params", [{"radius": 0.8, "range_filter": 1}]) - def test_milvus_client_search_iterator_with_l2_metric_type_with_params(self, metric_type, params): - """ - target: test search iterator with L2 metric type and search params - method: 1. search iterator - 2. check the result, expect pk - expected: search successfully - """ - client = self._client() - collection_name = cf.gen_unique_str(prefix) - self.using_database(client, "default") - # 1. create collection - self.create_collection(client, collection_name, default_dim, - metric_type=metric_type, consistency_level="Strong") - collections = self.list_collections(client)[0] - assert collection_name in collections - self.describe_collection(client, collection_name, - check_task=CheckTasks.check_describe_collection_property, - check_items={"collection_name": collection_name, - "dim": default_dim, - "consistency_level": 0}) - # 2. insert - rng = np.random.default_rng(seed=19530) - rows = [{default_primary_key_field_name: i, default_vector_field_name: list(rng.random((1, default_dim))[0]), - default_float_field_name: i * 1.0, default_string_field_name: str(i)} for i in range(default_nb)] - self.insert(client, collection_name, rows) - # 3. search - vectors_to_search = rng.random((1, default_dim)) - insert_ids = [i for i in range(default_nb)] - search_params = {"metric_type": metric_type, "params": params} - self.search_iterator(client, collection_name, vectors_to_search, - batch_size=100, - search_params=search_params, - check_task=CheckTasks.check_search_iterator, - check_items={"metric_type": metric_type, - "radius": 0.8, - "range_filter": 1}) - self.release_collection(client, collection_name) - self.drop_collection(client, collection_name) + check_items={"batch_size": batch_size}) @pytest.mark.tags(CaseLabel.L0) def test_milvus_client_search_iterator_external_filter_func_default(self): - pass + pass \ No newline at end of file diff --git a/tests/python_client/testcases/test_search.py b/tests/python_client/testcases/test_search.py index 0083172023..d85da326d5 100644 --- a/tests/python_client/testcases/test_search.py +++ b/tests/python_client/testcases/test_search.py @@ -4143,7 +4143,7 @@ class TestCollectionSearch(TestcaseBase): for t in threads: t.join() - @pytest.mark.tags(CaseLabel.L1) + @pytest.mark.tags(CaseLabel.L2) @pytest.mark.skip(reason="issue 37113") def test_search_concurrent_two_collections_nullable(self, nq, _async): """ @@ -4200,7 +4200,7 @@ class TestCollectionSearch(TestcaseBase): # 2. search with multi-processes log.info("test_search_concurrent_two_collections_nullable: searching with %s processes" % threads_num) for i in range(threads_num): - t = threading.Thread(target=search, args=(collection_w,)) + t = threading.Thread(target=search, args=(collection_w_1)) threads.append(t) t.start() time.sleep(0.2) @@ -10365,31 +10365,51 @@ class TestSearchIterator(TestcaseBase): """ Test case of search iterator """ @pytest.mark.tags(CaseLabel.L0) + @pytest.mark.parametrize("metric_type", ct.float_metrics) @pytest.mark.parametrize("vector_data_type", ["FLOAT_VECTOR", "FLOAT16_VECTOR", "BFLOAT16_VECTOR"]) - def test_search_iterator_normal(self, vector_data_type): + def test_range_search_iterator_default(self, metric_type, vector_data_type): """ - target: test search iterator normal + target: test iterator range search method: 1. search iterator - 2. check the result, expect pk + 2. check the result, expect pk not repeat and meet the range requirements expected: search successfully """ # 1. initialize with data - dim = 128 - collection_w = self.init_collection_general(prefix, True, dim=dim, is_index=False, + batch_size = 100 + collection_w = self.init_collection_general(prefix, True, dim=default_dim, is_index=False, vector_data_type=vector_data_type)[0] - collection_w.create_index(field_name, {"metric_type": "L2"}) + collection_w.create_index(field_name, {"metric_type": metric_type}) collection_w.load() + search_vector = cf.gen_vectors(1, default_dim, vector_data_type) + search_params = {"metric_type": metric_type} + collection_w.search_iterator(search_vector, field_name, search_params, batch_size, + check_task=CheckTasks.check_search_iterator, + check_items={"metric_type": metric_type, + "batch_size": batch_size}) + + limit = 200 + res = collection_w.search(search_vector, field_name, param=search_params, limit=200, + check_task=CheckTasks.check_search_results, + check_items={"nq": 1, "limit": limit})[0] # 2. search iterator - search_params = {"metric_type": "L2"} - vectors = cf.gen_vectors_based_on_vector_type(1, dim, vector_data_type) - batch_size = 200 - collection_w.search_iterator(vectors[:1], field_name, search_params, batch_size, - check_task=CheckTasks.check_search_iterator, - check_items={"batch_size": batch_size}) - batch_size = 600 - collection_w.search_iterator(vectors[:1], field_name, search_params, batch_size, - check_task=CheckTasks.check_search_iterator, - check_items={"batch_size": batch_size}) + if metric_type != "L2": + radius = res[0][limit // 2].distance - 0.1 # pick a radius to make sure there exists results + range_filter = res[0][0].distance + 0.1 + search_params = {"metric_type": metric_type, "params": {"radius": radius, "range_filter": range_filter}} + collection_w.search_iterator(search_vector, field_name, search_params, batch_size, + check_task=CheckTasks.check_search_iterator, + check_items={"metric_type": metric_type, "batch_size": batch_size, + "radius": radius, + "range_filter": range_filter}) + else: + radius = res[0][limit // 2].distance + 0.1 + range_filter = res[0][0].distance - 0.1 + search_params = {"metric_type": metric_type, "params": {"radius": radius, "range_filter": range_filter}} + collection_w.search_iterator(search_vector, field_name, search_params, batch_size, + check_task=CheckTasks.check_search_iterator, + check_items={"metric_type": metric_type, "batch_size": batch_size, + "radius": radius, + "range_filter": range_filter}) @pytest.mark.tags(CaseLabel.L1) def test_search_iterator_binary(self): @@ -10433,113 +10453,6 @@ class TestSearchIterator(TestcaseBase): expr=expression, check_task=CheckTasks.check_search_iterator, check_items={}) - @pytest.mark.tags(CaseLabel.L2) - def test_range_search_iterator_L2(self): - """ - target: test iterator range search - method: 1. search iterator - 2. check the result, expect pk not repeat and meet the expr requirements - expected: search successfully - """ - # 1. initialize with data - batch_size = 100 - collection_w = self.init_collection_general(prefix, True, is_index=False)[0] - collection_w.create_index(field_name, {"metric_type": "L2"}) - collection_w.load() - # 2. search iterator - search_params = {"metric_type": "L2", "params": {"radius": 35.0, "range_filter": 34.0}} - collection_w.search_iterator(vectors[:1], field_name, search_params, batch_size, - check_task=CheckTasks.check_search_iterator, - check_items={"metric_type": "L2", - "radius": 35.0, - "range_filter": 34.0}) - - @pytest.mark.tags(CaseLabel.L2) - def test_range_search_iterator_IP(self): - """ - target: test iterator range search - method: 1. search iterator - 2. check the result, expect pk not repeat and meet the expr requirements - expected: search successfully - """ - # 1. initialize with data - batch_size = 100 - collection_w = self.init_collection_general(prefix, True, is_index=False)[0] - collection_w.create_index(field_name, {"metric_type": "IP"}) - collection_w.load() - # 2. search iterator - search_params = {"metric_type": "IP", "params": {"radius": 0, "range_filter": 45}} - collection_w.search_iterator(vectors[:1], field_name, search_params, batch_size, - check_task=CheckTasks.check_search_iterator, - check_items={"metric_type": "IP", - "radius": 0, - "range_filter": 45}) - - @pytest.mark.tags(CaseLabel.L1) - def test_range_search_iterator_COSINE(self): - """ - target: test iterator range search - method: 1. search iterator - 2. check the result, expect pk not repeat and meet the expr requirements - expected: search successfully - """ - # 1. initialize with data - batch_size = 100 - collection_w = self.init_collection_general(prefix, True, is_index=False)[0] - collection_w.create_index(field_name, {"metric_type": "COSINE"}) - collection_w.load() - # 2. search iterator - search_params = {"metric_type": "COSINE", "params": {"radius": 0.8, "range_filter": 1}} - collection_w.search_iterator(vectors[:1], field_name, search_params, batch_size, - check_task=CheckTasks.check_search_iterator, - check_items={"metric_type": "COSINE", - "radius": 0.8, - "range_filter": 1}) - - @pytest.mark.tags(CaseLabel.L2) - def test_range_search_iterator_only_radius(self): - """ - target: test search iterator normal - method: 1. search iterator - 2. check the result, expect pk not repeat and meet the expr requirements - expected: search successfully - """ - # 1. initialize with data - batch_size = 100 - collection_w = self.init_collection_general(prefix, True, is_index=False)[0] - collection_w.create_index(field_name, {"metric_type": "L2"}) - collection_w.load() - # 2. search iterator - search_params = {"metric_type": "L2", "params": {"radius": 35.0}} - collection_w.search_iterator(vectors[:1], field_name, search_params, batch_size, - check_task=CheckTasks.check_search_iterator, - check_items={"metric_type": "L2", - "radius": 35.0}) - - @pytest.mark.tags(CaseLabel.L2) - @pytest.mark.skip("issue #25145") - @pytest.mark.parametrize("index", ct.all_index_types[:7]) - @pytest.mark.parametrize("metrics", ct.float_metrics) - def test_search_iterator_after_different_index_metrics(self, index, metrics): - """ - target: test search iterator using different index - method: 1. search iterator - 2. check the result, expect pk not repeat and meet the expr requirements - expected: search successfully - """ - # 1. initialize with data - batch_size = 100 - collection_w = self.init_collection_general(prefix, True, is_index=False)[0] - params = cf.get_index_params_params(index) - default_index = {"index_type": index, "params": params, "metric_type": metrics} - collection_w.create_index(field_name, default_index) - collection_w.load() - # 2. search iterator - search_params = {"metric_type": metrics} - collection_w.search_iterator(vectors[:1], field_name, search_params, batch_size, - check_task=CheckTasks.check_search_iterator, - check_items={}) - @pytest.mark.tags(CaseLabel.L2) @pytest.mark.parametrize("batch_size", [10, 100, 777, 1000]) def test_search_iterator_with_different_limit(self, batch_size):