admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 11:21:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

Load collection manually

Browse Source 
Signed-off-by: neza2017 <yefu.chen@zilliz.com>
This commit is contained in:
neza2017 2021-02-24 10:35:08 +08:00 committed by yefu.chen
parent 000e30d2cf
commit 2b32e6c912
8 changed files with 838 additions and 1283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tests/python/conftest.py
View File

@ -171,6 +171,7 @@ def collection(request, connect):
try:
default_fields = gen_default_fields()
connect.create_collection(collection_name, default_fields)
connect.load_collection(collection_name)
except Exception as e:
pytest.exit(str(e))
def teardown():
@ -189,6 +190,7 @@ def id_collection(request, connect):
try:
fields = gen_default_fields(auto_id=False)
connect.create_collection(collection_name, fields)
connect.load_collection(collection_name)
except Exception as e:
pytest.exit(str(e))
def teardown():
@ -206,6 +208,7 @@ def binary_collection(request, connect):
try:
fields = gen_binary_default_fields()
connect.create_collection(collection_name, fields)
connect.load_collection(collection_name)
except Exception as e:
pytest.exit(str(e))
def teardown():
@ -225,6 +228,7 @@ def binary_id_collection(request, connect):
try:
fields = gen_binary_default_fields(auto_id=False)
connect.create_collection(collection_name, fields)
connect.load_collection(collection_name)
except Exception as e:
pytest.exit(str(e))
def teardown():

1148
tests/python/test_bulk_insert.py
View File

File diff suppressed because it is too large Load Diff

4
tests/python/test_create_collection.py
View File

@ -97,7 +97,7 @@ class TestCreateCollection:
expected: error raised
'''
# pdb.set_trace()
connect.bulk_insert(collection, default_entity)
connect.insert(collection, default_entity)
with pytest.raises(Exception) as e:
connect.create_collection(collection, default_fields)
@ -108,7 +108,7 @@ class TestCreateCollection:
method: insert vector and create collection
expected: error raised
'''
connect.bulk_insert(collection, default_entity)
connect.insert(collection, default_entity)
connect.flush([collection])
with pytest.raises(Exception) as e:
connect.create_collection(collection, default_fields)

12
tests/python/test_get_collection_info.py
View File

@ -33,9 +33,9 @@ class TestInfoBase:
)
def get_simple_index(self, request, connect):
logging.getLogger().info(request.param)
if str(connect._cmd("mode")) == "CPU":
if request.param["index_type"] in index_cpu_not_support():
pytest.skip("sq8h not support in CPU mode")
# if str(connect._cmd("mode")) == "CPU":
if request.param["index_type"] in index_cpu_not_support():
pytest.skip("sq8h not support in CPU mode")
return request.param
"""
@ -88,7 +88,7 @@ class TestInfoBase:
@pytest.mark.skip("no create Index")
def test_get_collection_info_after_index_created(self, connect, collection, get_simple_index):
connect.create_index(collection, default_float_vec_field_name, get_simple_index)
info = connect.get_index_info(collection, field_name)
info = connect.describe_index(collection, field_name)
assert info == get_simple_index
res = connect.get_collection_info(collection, default_float_vec_field_name)
assert index["index_type"] == get_simple_index["index_type"]
@ -161,7 +161,7 @@ class TestInfoBase:
}
connect.create_collection(collection_name, fields)
entities = gen_entities_by_fields(fields["fields"], default_nb, vector_field["params"]["dim"])
res_ids = connect.bulk_insert(collection_name, entities)
res_ids = connect.insert(collection_name, entities)
connect.flush([collection_name])
res = connect.get_collection_info(collection_name)
assert res['auto_id'] == True
@ -186,7 +186,7 @@ class TestInfoBase:
fields["segment_row_limit"] = get_segment_row_limit
connect.create_collection(collection_name, fields)
entities = gen_entities_by_fields(fields["fields"], default_nb, fields["fields"][-1]["params"]["dim"])
res_ids = connect.bulk_insert(collection_name, entities)
res_ids = connect.insert(collection_name, entities)
connect.flush([collection_name])
res = connect.get_collection_info(collection_name)
assert res['auto_id'] == True

66
tests/python/test_index.py
View File

@ -26,9 +26,9 @@ class TestIndexBase:
def get_simple_index(self, request, connect):
import copy
logging.getLogger().info(request.param)
if str(connect._cmd("mode")) == "CPU":
if request.param["index_type"] in index_cpu_not_support():
pytest.skip("sq8h not support in CPU mode")
#if str(connect._cmd("mode")) == "CPU":
if request.param["index_type"] in index_cpu_not_support():
pytest.skip("sq8h not support in CPU mode")
return copy.deepcopy(request.param)
@pytest.fixture(
@ -55,7 +55,7 @@ class TestIndexBase:
method: create collection and add entities in it, create index
expected: return search success
'''
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
connect.create_index(collection, field_name, get_simple_index)
def test_create_index_on_field_not_existed(self, connect, collection, get_simple_index):
@ -65,7 +65,7 @@ class TestIndexBase:
expected: error raised
'''
tmp_field_name = gen_unique_str()
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
with pytest.raises(Exception) as e:
connect.create_index(collection, tmp_field_name, get_simple_index)
@ -77,7 +77,7 @@ class TestIndexBase:
expected: error raised
'''
tmp_field_name = "int64"
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
with pytest.raises(Exception) as e:
connect.create_index(collection, tmp_field_name, get_simple_index)
@ -98,7 +98,7 @@ class TestIndexBase:
expected: return search success
'''
connect.create_partition(collection, default_tag)
ids = connect.bulk_insert(collection, default_entities, partition_tag=default_tag)
ids = connect.insert(collection, default_entities, partition_tag=default_tag)
connect.flush([collection])
connect.create_index(collection, field_name, get_simple_index)
@ -110,7 +110,7 @@ class TestIndexBase:
expected: return search success
'''
connect.create_partition(collection, default_tag)
ids = connect.bulk_insert(collection, default_entities, partition_tag=default_tag)
ids = connect.insert(collection, default_entities, partition_tag=default_tag)
connect.flush()
connect.create_index(collection, field_name, get_simple_index)
@ -131,7 +131,7 @@ class TestIndexBase:
method: create collection and add entities in it, create index
expected: return search success
'''
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
connect.create_index(collection, field_name, get_simple_index)
# logging.getLogger().info(connect.get_collection_stats(collection))
nq = get_nq
@ -150,7 +150,7 @@ class TestIndexBase:
method: create collection and add entities in it, create index
expected: return search success
'''
connect.bulk_insert(collection, default_entities)
connect.insert(collection, default_entities)
def build(connect):
connect.create_index(collection, field_name, default_index)
@ -187,7 +187,7 @@ class TestIndexBase:
expected: create index ok, and count correct
'''
connect.create_index(collection, field_name, get_simple_index)
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
connect.flush([collection])
count = connect.count_entities(collection)
assert count == default_nb
@ -213,7 +213,7 @@ class TestIndexBase:
method: create another index with different index_params after index have been built
expected: return code 0, and describe index result equals with the second index params
'''
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
indexs = [default_index, {"metric_type":"L2", "index_type": "FLAT", "params":{"nlist": 1024}}]
for index in indexs:
connect.create_index(collection, field_name, index)
@ -228,7 +228,7 @@ class TestIndexBase:
method: create collection and add entities in it, create index
expected: return search success
'''
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
get_simple_index["metric_type"] = "IP"
connect.create_index(collection, field_name, get_simple_index)
@ -250,7 +250,7 @@ class TestIndexBase:
expected: return search success
'''
connect.create_partition(collection, default_tag)
ids = connect.bulk_insert(collection, default_entities, partition_tag=default_tag)
ids = connect.insert(collection, default_entities, partition_tag=default_tag)
connect.flush([collection])
get_simple_index["metric_type"] = "IP"
connect.create_index(collection, field_name, get_simple_index)
@ -263,7 +263,7 @@ class TestIndexBase:
expected: return search success
'''
connect.create_partition(collection, default_tag)
ids = connect.bulk_insert(collection, default_entities, partition_tag=default_tag)
ids = connect.insert(collection, default_entities, partition_tag=default_tag)
connect.flush()
get_simple_index["metric_type"] = "IP"
connect.create_index(collection, field_name, get_simple_index)
@ -277,7 +277,7 @@ class TestIndexBase:
expected: return search success
'''
metric_type = "IP"
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
get_simple_index["metric_type"] = metric_type
connect.create_index(collection, field_name, get_simple_index)
# logging.getLogger().info(connect.get_collection_stats(collection))
@ -297,7 +297,7 @@ class TestIndexBase:
method: create collection and add entities in it, create index
expected: return search success
'''
connect.bulk_insert(collection, default_entities)
connect.insert(collection, default_entities)
def build(connect):
default_index["metric_type"] = "IP"
@ -336,7 +336,7 @@ class TestIndexBase:
'''
default_index["metric_type"] = "IP"
connect.create_index(collection, field_name, get_simple_index)
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
connect.flush([collection])
count = connect.count_entities(collection)
assert count == default_nb
@ -364,7 +364,7 @@ class TestIndexBase:
method: create another index with different index_params after index have been built
expected: return code 0, and describe index result equals with the second index params
'''
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
indexs = [default_index, {"index_type": "FLAT", "params": {"nlist": 1024}, "metric_type": "IP"}]
for index in indexs:
connect.create_index(collection, field_name, index)
@ -385,7 +385,7 @@ class TestIndexBase:
method: create collection and add entities in it, create index, call drop index
expected: return code 0, and default index param
'''
# ids = connect.bulk_insert(collection, entities)
# ids = connect.insert(collection, entities)
connect.create_index(collection, field_name, get_simple_index)
connect.drop_index(collection, field_name)
stats = connect.get_collection_stats(collection)
@ -439,7 +439,7 @@ class TestIndexBase:
method: create collection and add entities in it, create index
expected: return code not equals to 0, drop index failed
'''
# ids = connect.bulk_insert(collection, entities)
# ids = connect.insert(collection, entities)
# no create index
connect.drop_index(collection, field_name)
@ -462,7 +462,7 @@ class TestIndexBase:
method: create collection and add entities in it, create index, call drop index
expected: return code 0, and default index param
'''
# ids = connect.bulk_insert(collection, entities)
# ids = connect.insert(collection, entities)
get_simple_index["metric_type"] = "IP"
connect.create_index(collection, field_name, get_simple_index)
connect.drop_index(collection, field_name)
@ -506,7 +506,7 @@ class TestIndexBase:
method: create collection and add entities in it, create index
expected: return code not equals to 0, drop index failed
'''
# ids = connect.bulk_insert(collection, entities)
# ids = connect.insert(collection, entities)
# no create index
connect.drop_index(collection, field_name)
@ -579,7 +579,7 @@ class TestIndexBinary:
method: create collection and add entities in it, create index
expected: return search success
'''
ids = connect.bulk_insert(binary_collection, default_binary_entities)
ids = connect.insert(binary_collection, default_binary_entities)
connect.create_index(binary_collection, binary_field_name, get_jaccard_index)
@pytest.mark.timeout(BUILD_TIMEOUT)
@ -590,7 +590,7 @@ class TestIndexBinary:
expected: return search success
'''
connect.create_partition(binary_collection, default_tag)
ids = connect.bulk_insert(binary_collection, default_binary_entities, partition_tag=default_tag)
ids = connect.insert(binary_collection, default_binary_entities, partition_tag=default_tag)
connect.create_index(binary_collection, binary_field_name, get_jaccard_index)
@pytest.mark.skip("r0.3-test")
@ -602,7 +602,7 @@ class TestIndexBinary:
expected: return search success
'''
nq = get_nq
ids = connect.bulk_insert(binary_collection, default_binary_entities)
ids = connect.insert(binary_collection, default_binary_entities)
connect.create_index(binary_collection, binary_field_name, get_jaccard_index)
query, vecs = gen_query_vectors(binary_field_name, default_binary_entities, default_top_k, nq, metric_type="JACCARD")
search_param = get_search_param(get_jaccard_index["index_type"], metric_type="JACCARD")
@ -619,7 +619,7 @@ class TestIndexBinary:
expected: return create_index failure
'''
# insert 6000 vectors
ids = connect.bulk_insert(binary_collection, default_binary_entities)
ids = connect.insert(binary_collection, default_binary_entities)
connect.flush([binary_collection])
if get_l2_index["index_type"] == "BIN_FLAT":
@ -641,7 +641,7 @@ class TestIndexBinary:
method: create collection and add entities in it, create index, call describe index
expected: return code 0, and index instructure
'''
ids = connect.bulk_insert(binary_collection, default_binary_entities)
ids = connect.insert(binary_collection, default_binary_entities)
connect.flush([binary_collection])
connect.create_index(binary_collection, binary_field_name, get_jaccard_index)
stats = connect.get_collection_stats(binary_collection)
@ -662,7 +662,7 @@ class TestIndexBinary:
expected: return code 0, and index instructure
'''
connect.create_partition(binary_collection, default_tag)
ids = connect.bulk_insert(binary_collection, default_binary_entities, partition_tag=default_tag)
ids = connect.insert(binary_collection, default_binary_entities, partition_tag=default_tag)
connect.flush([binary_collection])
connect.create_index(binary_collection, binary_field_name, get_jaccard_index)
stats = connect.get_collection_stats(binary_collection)
@ -706,7 +706,7 @@ class TestIndexBinary:
expected: return code 0, and default index param
'''
connect.create_partition(binary_collection, default_tag)
ids = connect.bulk_insert(binary_collection, default_binary_entities, partition_tag=default_tag)
ids = connect.insert(binary_collection, default_binary_entities, partition_tag=default_tag)
connect.flush([binary_collection])
connect.create_index(binary_collection, binary_field_name, get_jaccard_index)
stats = connect.get_collection_stats(binary_collection)
@ -802,7 +802,7 @@ class TestIndexAsync:
method: create collection and add entities in it, create index
expected: return search success
'''
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
logging.getLogger().info("start index")
future = connect.create_index(collection, field_name, get_simple_index, _async=True)
logging.getLogger().info("before result")
@ -817,7 +817,7 @@ class TestIndexAsync:
method: create collection and add entities in it, create index
expected: return search success
'''
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
logging.getLogger().info("start index")
future = connect.create_index(collection, field_name, get_simple_index, _async=True)
logging.getLogger().info("DROP")
@ -837,7 +837,7 @@ class TestIndexAsync:
method: create collection and add entities in it, create index
expected: return search success
'''
ids = connect.bulk_insert(collection, default_entities)
ids = connect.insert(collection, default_entities)
logging.getLogger().info("start index")
future = connect.create_index(collection, field_name, get_simple_index, _async=True,
_callback=self.check_result)

841
tests/python/test_insert.py
View File

File diff suppressed because it is too large Load Diff

14
tests/python/test_partition.py
View File

@ -101,7 +101,7 @@ class TestCreateBase:
'''
connect.create_partition(id_collection, default_tag)
ids = [i for i in range(default_nb)]
insert_ids = connect.bulk_insert(id_collection, default_entities, ids)
insert_ids = connect.insert(id_collection, default_entities, ids)
assert len(insert_ids) == len(ids)
@pytest.mark.skip("not support custom id")
@ -113,7 +113,7 @@ class TestCreateBase:
'''
connect.create_partition(id_collection, default_tag)
ids = [i for i in range(default_nb)]
insert_ids = connect.bulk_insert(id_collection, default_entities, ids, partition_tag=default_tag)
insert_ids = connect.insert(id_collection, default_entities, ids, partition_tag=default_tag)
assert len(insert_ids) == len(ids)
def test_create_partition_insert_with_tag_not_existed(self, connect, collection):
@ -126,7 +126,7 @@ class TestCreateBase:
connect.create_partition(collection, default_tag)
ids = [i for i in range(default_nb)]
with pytest.raises(Exception) as e:
insert_ids = connect.bulk_insert(collection, default_entities, ids, partition_tag=tag_new)
insert_ids = connect.insert(collection, default_entities, ids, partition_tag=tag_new)
@pytest.mark.skip("not support custom id")
def test_create_partition_insert_same_tags(self, connect, id_collection):
@ -137,9 +137,9 @@ class TestCreateBase:
'''
connect.create_partition(id_collection, default_tag)
ids = [i for i in range(default_nb)]
insert_ids = connect.bulk_insert(id_collection, default_entities, ids, partition_tag=default_tag)
insert_ids = connect.insert(id_collection, default_entities, ids, partition_tag=default_tag)
ids = [(i+default_nb) for i in range(default_nb)]
new_insert_ids = connect.bulk_insert(id_collection, default_entities, ids, partition_tag=default_tag)
new_insert_ids = connect.insert(id_collection, default_entities, ids, partition_tag=default_tag)
connect.flush([id_collection])
res = connect.count_entities(id_collection)
assert res == default_nb * 2
@ -156,8 +156,8 @@ class TestCreateBase:
collection_new = gen_unique_str()
connect.create_collection(collection_new, default_fields)
connect.create_partition(collection_new, default_tag)
ids = connect.bulk_insert(collection, default_entities, partition_tag=default_tag)
ids = connect.bulk_insert(collection_new, default_entities, partition_tag=default_tag)
ids = connect.insert(collection, default_entities, partition_tag=default_tag)
ids = connect.insert(collection_new, default_entities, partition_tag=default_tag)
connect.flush([collection, collection_new])
res = connect.count_entities(collection)
assert res == default_nb

32
tests/python/test_search.py
View File

@ -36,14 +36,14 @@ def init_data(connect, collection, nb=1200, partition_tags=None, auto_id=True):
insert_entities = gen_entities(nb, is_normal=True)
if partition_tags is None:
if auto_id:
ids = connect.bulk_insert(collection, insert_entities)
ids = connect.insert(collection, insert_entities)
else:
ids = connect.bulk_insert(collection, insert_entities, ids=[i for i in range(nb)])
ids = connect.insert(collection, insert_entities, ids=[i for i in range(nb)])
else:
if auto_id:
ids = connect.bulk_insert(collection, insert_entities, partition_tag=partition_tags)
ids = connect.insert(collection, insert_entities, partition_tag=partition_tags)
else:
ids = connect.bulk_insert(collection, insert_entities, ids=[i for i in range(nb)], partition_tag=partition_tags)
ids = connect.insert(collection, insert_entities, ids=[i for i in range(nb)], partition_tag=partition_tags)
# connect.flush([collection])
return insert_entities, ids
@ -62,9 +62,9 @@ def init_binary_data(connect, collection, nb=1200, insert=True, partition_tags=N
insert_raw_vectors, insert_entities = gen_binary_entities(nb)
if insert is True:
if partition_tags is None:
ids = connect.bulk_insert(collection, insert_entities)
ids = connect.insert(collection, insert_entities)
else:
ids = connect.bulk_insert(collection, insert_entities, partition_tag=partition_tags)
ids = connect.insert(collection, insert_entities, partition_tag=partition_tags)
connect.flush([collection])
return insert_raw_vectors, insert_entities, ids
@ -79,9 +79,9 @@ class TestSearchBase:
params=gen_index()
)
def get_index(self, request, connect):
if str(connect._cmd("mode")) == "CPU":
if request.param["index_type"] in index_cpu_not_support():
pytest.skip("sq8h not support in CPU mode")
# if str(connect._cmd("mode")) == "CPU":
if request.param["index_type"] in index_cpu_not_support():
pytest.skip("sq8h not support in CPU mode")
return request.param
@pytest.fixture(
@ -90,9 +90,9 @@ class TestSearchBase:
)
def get_simple_index(self, request, connect):
import copy
if str(connect._cmd("mode")) == "CPU":
if request.param["index_type"] in index_cpu_not_support():
pytest.skip("sq8h not support in CPU mode")
# if str(connect._cmd("mode")) == "CPU":
if request.param["index_type"] in index_cpu_not_support():
pytest.skip("sq8h not support in CPU mode")
return copy.deepcopy(request.param)
@pytest.fixture(
@ -1250,7 +1250,7 @@ class TestSearchDSL(object):
collection_term = gen_unique_str("term")
connect.create_collection(collection_term, term_fields)
term_entities = add_field(entities, field_name="term")
ids = connect.bulk_insert(collection_term, term_entities)
ids = connect.insert(collection_term, term_entities)
assert len(ids) == default_nb
connect.flush([collection_term])
count = connect.count_entities(collection_term) # count_entities is not impelmented
@ -1695,9 +1695,9 @@ class TestSearchInvalid(object):
params=gen_simple_index()
)
def get_simple_index(self, request, connect):
if str(connect._cmd("mode")) == "CPU":
if request.param["index_type"] in index_cpu_not_support():
pytest.skip("sq8h not support in CPU mode")
# if str(connect._cmd("mode")) == "CPU":
if request.param["index_type"] in index_cpu_not_support():
pytest.skip("sq8h not support in CPU mode")
return request.param
# PASS