admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-28 22:45:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #686 from milvus-io/0.6.0

Browse Source 
Merge from 0.6.0
This commit is contained in:
Jin Hai 2019-12-04 14:25:48 +08:00 committed by GitHub
commit df61c98aa3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 1659 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.all-contributorsrc
View File

@ -5,6 +5,15 @@
"imageSize": 100,
"commit": false,
"contributors": [
{
"login": "gaolizhou",
"name": "gaolizhou",
"avatar_url": "https://avatars2.githubusercontent.com/u/2884044?v=4",
"profile": "https://github.com/gaolizhou",
"contributions": [
"code"
]
},
{
"login": "akihoni",
"name": "Sijie Zhang",

1
README.md
View File

@ -62,6 +62,7 @@ Below is a list of Milvus contributors. We greatly appreciate your contributions
<!-- markdownlint-disable -->
<table>
<tr>
<td align="center"><a href="https://github.com/gaolizhou"><img src="https://avatars2.githubusercontent.com/u/2884044?v=4" width="100px;" alt="gaolizhou"/><br /><sub><b>gaolizhou</b></sub></a><br /><a href="https://github.com/milvus-io/milvus/commits?author=gaolizhou" title="Code">💻</a></td>
<td align="center"><a href="https://github.com/akihoni"><img src="https://avatars0.githubusercontent.com/u/36330442?v=4" width="100px;" alt="Sijie Zhang"/><br /><sub><b>Sijie Zhang</b></sub></a><br /><a href="https://github.com/milvus-io/milvus/commits?author=akihoni" title="Documentation">📖</a></td>
</tr>
</table>

6
tests/milvus_ann_acc/client.py
View File

@ -88,8 +88,10 @@ class MilvusClient(object):
index_type = IndexType.IVF_SQ8
elif index_type == "ivf_sq8h":
index_type = IndexType.IVF_SQ8H
elif index_type == "mix_nsg":
index_type = IndexType.MIX_NSG
elif index_type == "nsg":
index_type = IndexType.NSG
elif index_type == "ivf_pq":
index_type = IndexType.IVF_PQ
index_params = {
"index_type": index_type,
"nlist": nlist,

6
tests/milvus_ann_acc/requirements.txt
View File

@ -1,9 +1,9 @@
numpy==1.16.3
pymilvus>=0.2.0
pymilvus-test>=0.2.0
scikit-learn==0.19.1
h5py==2.7.1
influxdb==5.2.2
pyyaml==3.12
pyyaml>=5.1
tableprint==0.8.0
ansicolors==1.1.8
scipy==1.3.1
scipy==1.3.1

6
tests/milvus_ann_acc/runner.py
View File

@ -55,9 +55,9 @@ def recall_calc(result_ids, true_ids, top_k, recall_k):
sum_intersect_num = 0
recall = 0.0
for index, result_item in enumerate(result_ids):
if len(set(true_ids[index][:top_k])) != len(set(result_item)):
logger.warning("Error happened: query result length is wrong")
continue
# if len(set(true_ids[index][:top_k])) != len(set(result_item)):
# logger.warning("Error happened: query result length is wrong")
# continue
tmp = set(true_ids[index][:recall_k]).intersection(set(result_item))
sum_intersect_num = sum_intersect_num + len(tmp)
recall = round(sum_intersect_num / (len(result_ids) * recall_k), 4)

14
tests/milvus_ann_acc/suite.yaml
View File

@ -1,29 +1,29 @@
datasets:
- sift-128-euclidean:
index_file_sizes: [50, 1024]
index_file_sizes: [1024]
index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h']
# index_types: ['ivf_sq8']
nlists: [16384]
search_param:
nprobes: [1, 32, 128, 256]
nprobes: [1, 8, 16, 32, 64, 128, 256, 512]
top_ks: [10]
nqs: [10000]
- glove-25-angular:
index_file_sizes: [50, 1024]
index_file_sizes: [1024]
index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h']
# index_types: ['ivf_sq8']
nlists: [16384]
search_param:
nprobes: [1, 32, 128, 256]
nprobes: [1, 8, 16, 32, 64, 128, 256, 512]
top_ks: [10]
nqs: [10000]
- glove-200-angular:
index_file_sizes: [50, 1024]
index_file_sizes: [1024]
index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h']
# index_types: ['ivf_sq8']
nlists: [16384]
search_param:
nprobes: [1, 32, 128, 256]
nprobes: [1, 8, 16, 32, 64, 128, 256, 512]
top_ks: [10]
nqs: [10000]
hdf5_path: /test/milvus/ann_hdf5/
hdf5_path: /test/milvus/ann_hdf5/

19
tests/milvus_ann_acc/suite_cpu.yaml Normal file
View File

@ -0,0 +1,19 @@
datasets:
- sift-128-euclidean:
index_file_sizes: [1024]
index_types: ['ivf_flat', 'ivf_sq8']
nlists: [16384]
search_param:
nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
top_ks: [10]
nqs: [10000]
- glove-200-angular:
index_file_sizes: [1024]
index_types: ['ivf_flat', 'ivf_sq8']
# index_types: ['ivf_sq8']
nlists: [16384]
search_param:
nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
top_ks: [10]
nqs: [10000]
hdf5_path: /test/milvus/ann_hdf5/

13
tests/milvus_ann_acc/suite_debug.yaml
View File

@ -1,10 +1,19 @@
datasets:
- sift-128-euclidean:
index_file_sizes: [1024]
index_types: ['ivf_flat']
index_types: ['ivf_flat', 'ivf_sq8']
nlists: [16384]
search_param:
nprobes: [1, 256]
nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
top_ks: [10]
nqs: [10000]
- glove-200-angular:
index_file_sizes: [1024]
index_types: ['ivf_flat', 'ivf_sq8']
# index_types: ['ivf_sq8']
nlists: [16384]
search_param:
nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
top_ks: [10]
nqs: [10000]
hdf5_path: /test/milvus/ann_hdf5/

19
tests/milvus_ann_acc/suite_gpu.yaml Normal file
View File

@ -0,0 +1,19 @@
datasets:
- sift-128-euclidean:
index_file_sizes: [1024]
index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'ivf_pq']
nlists: [16384]
search_param:
nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
top_ks: [10]
nqs: [10000]
- glove-200-angular:
index_file_sizes: [1024]
index_types: ['ivf_flat', 'ivf_sq8']
# index_types: ['ivf_sq8']
nlists: [16384]
search_param:
nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
top_ks: [10]
nqs: [10000]
hdf5_path: /test/milvus/ann_hdf5/

127
tests/milvus_benchmark/client.py
View File

@ -10,6 +10,7 @@ from milvus import Milvus, IndexType, MetricType
logger = logging.getLogger("milvus_benchmark.client")
SERVER_HOST_DEFAULT = "127.0.0.1"
# SERVER_HOST_DEFAULT = "192.168.1.130"
SERVER_PORT_DEFAULT = 19530
@ -80,16 +81,18 @@ class MilvusClient(object):
index_type = IndexType.IVFLAT
elif index_type == "ivf_sq8":
index_type = IndexType.IVF_SQ8
elif index_type == "mix_nsg":
index_type = IndexType.MIX_NSG
elif index_type == "nsg":
index_type = IndexType.NSG
elif index_type == "ivf_sq8h":
index_type = IndexType.IVF_SQ8H
elif index_type == "ivf_pq":
index_type = IndexType.IVF_PQ
index_params = {
"index_type": index_type,
"nlist": nlist,
}
logger.info("Building index start, table_name: %s, index_params: %s" % (self._table_name, json.dumps(index_params)))
status = self._milvus.create_index(self._table_name, index=index_params, timeout=6*3600)
status = self._milvus.create_index(self._table_name, index=index_params)
self.check_status(status)
def describe_index(self):
@ -125,8 +128,13 @@ class MilvusClient(object):
def describe(self):
return self._milvus.describe_table(self._table_name)
def show_tables(self):
return self._milvus.show_tables()
def exists_table(self):
return self._milvus.has_table(self._table_name)
status, res = self._milvus.has_table(self._table_name)
self.check_status(status)
return res
@time_wrapper
def preload_table(self):
@ -154,34 +162,60 @@ def fit_concurrent(table_name, process_num, vectors):
if __name__ == "__main__":
import numpy
import sklearn.preprocessing
# table_name = "sift_2m_20_128_l2"
table_name = "test_tset1"
m = MilvusClient(table_name)
# m.create_table(table_name, 128, 50, "l2")
# table_name = "tset_test"
# # table_name = "test_tset1"
# m = MilvusClient(table_name)
# m.delete()
# time.sleep(2)
# m.create_table(table_name, 128, 20, "ip")
print(m.describe())
# print(m.describe())
# print(m.count())
# print(m.describe_index())
insert_vectors = [[random.random() for _ in range(128)] for _ in range(10000)]
for i in range(5):
m.insert(insert_vectors)
print(m.create_index("ivf_sq8h", 16384))
X = [insert_vectors[0]]
top_k = 10
nprobe = 10
print(m.query(X, top_k, nprobe))
# # # print(m.drop_index())
# # print(m.describe_index())
# # sys.exit()
# # # insert_vectors = [[random.random() for _ in range(128)] for _ in range(100000)]
# # # for i in range(100):
# # # m.insert(insert_vectors)
# # # time.sleep(5)
# # # print(m.describe_index())
# # # print(m.drop_index())
# # m.create_index("ivf_sq8h", 16384)
# tmp = [[random.random() for _ in range(128)] for _ in range(20000)]
# tmp1 = sklearn.preprocessing.normalize(tmp, axis=1, norm='l2')
# print(tmp1[0][0])
# tmp = [[random.random() for _ in range(128)] for _ in range(20000)]
# tmp /= numpy.linalg.norm(tmp)
# print(tmp[0][0])
# sum_1 = 0
# sum_2 = 0
# for item in tmp:
# for i in item:
# sum_2 = sum_2 + i * i
# break
# for item in tmp1:
# for i in item:
# sum_1 = sum_1 + i * i
# break
# print(sum_1, sum_2)
# insert_vectors = tmp.tolist()
# # print(insert_vectors)
# for i in range(2):
# m.insert(insert_vectors)
# time.sleep(5)
# print(m.create_index("ivf_flat", 16384))
# X = [insert_vectors[0], insert_vectors[1], insert_vectors[2]]
# top_k = 5
# nprobe = 1
# print(m.query(X, top_k, nprobe))
# # print(m.drop_index())
# print(m.describe_index())
# sys.exit()
# # insert_vectors = [[random.random() for _ in range(128)] for _ in range(100000)]
# # for i in range(100):
# # m.insert(insert_vectors)
# # time.sleep(5)
# # print(m.describe_index())
# # print(m.drop_index())
# m.create_index("ivf_sq8h", 16384)
# print(m.count())
# print(m.describe_index())
@ -193,12 +227,12 @@ if __name__ == "__main__":
# print(m.describe_index())
import numpy as np
def mmap_fvecs(fname):
x = np.memmap(fname, dtype='int32', mode='r')
d = x[0]
return x.view('float32').reshape(-1, d + 1)[:, 1:]
# def mmap_fvecs(fname):
# x = np.memmap(fname, dtype='int32', mode='r')
# d = x[0]
# return x.view('float32').reshape(-1, d + 1)[:, 1:]
print(mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs"))
# print(mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs"))
# SIFT_SRC_QUERY_DATA_DIR = '/poc/yuncong/ann_1000m'
# file_name = SIFT_SRC_QUERY_DATA_DIR+'/'+'query.npy'
# data = numpy.load(file_name)
@ -229,16 +263,21 @@ if __name__ == "__main__":
# return x.view('float32').reshape(-1, d + 1)[:, 1:]
# data = mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs")
# print(data[0], len(data[0]), len(data))
# data = sklearn.preprocessing.normalize(data, axis=1, norm='l2')
# np.save("/test/milvus/deep1b/query.npy", data)
# total_size = 10000
# # total_size = 1000000000
# file_size = 1000
# # file_size = 100000
# file_num = total_size // file_size
# for i in range(file_num):
# fname = "/test/milvus/raw_data/deep1b/binary_96_%05d" % i
# print(fname, i*file_size, (i+1)*file_size)
# single_data = data[i*file_size : (i+1)*file_size]
# single_data = sklearn.preprocessing.normalize(single_data, axis=1, norm='l2')
# np.save(fname, single_data)
total_size = 100000000
# total_size = 1000000000
file_size = 100000
# file_size = 100000
dimension = 4096
file_num = total_size // file_size
for i in range(file_num):
print(i)
# fname = "/test/milvus/raw_data/deep1b/binary_96_%05d" % i
fname = "/test/milvus/raw_data/random/binary_%dd_%05d" % (dimension, i)
# print(fname, i*file_size, (i+1)*file_size)
# single_data = data[i*file_size : (i+1)*file_size]
single_data = [[random.random() for _ in range(dimension)] for _ in range(file_size)]
single_data = sklearn.preprocessing.normalize(single_data, axis=1, norm='l2')
np.save(fname, single_data)

60
tests/milvus_benchmark/conf/server_config.yaml
View File

@ -1,28 +1,34 @@
cache_config:
cache_insert_data: false
cpu_cache_capacity: 16
gpu_cache_capacity: 6
cpu_cache_threshold: 0.85
db_config:
backend_url: sqlite://:@:/
build_index_gpu: 0
insert_buffer_size: 4
preload_table: null
primary_path: /opt/milvus
secondary_path: null
engine_config:
use_blas_threshold: 20
metric_config:
collector: prometheus
enable_monitor: true
prometheus_config:
port: 8080
resource_config:
resource_pool:
- cpu
- gpu0
server_config:
address: 0.0.0.0
deploy_mode: single
port: 19530
time_zone: UTC+8
address: 0.0.0.0
port: 19530
deploy_mode: single
time_zone: UTC+8
db_config:
primary_path: /var/lib/milvus
secondary_path:
backend_url: sqlite://:@:/
insert_buffer_size: 4
preload_table:
metric_config:
enable_monitor: true
collector: prometheus
prometheus_config:
port: 8080
cache_config:
cpu_cache_capacity: 16
cache_insert_data: false
engine_config:
use_blas_threshold: 1100
gpu_search_threshold: 1000
gpu_resource_config:
enable: true
cache_capacity: 4
search_resources:
- gpu0
build_index_resources:
- gpu0

34
tests/milvus_benchmark/docker_runner.py
View File

@ -28,6 +28,9 @@ class DockerRunner(Runner):
container = None
if op_type == "insert":
if not run_params:
logger.debug("No run params")
continue
for index, param in enumerate(run_params):
logger.info("Definition param: %s" % str(param))
table_name = param["table_name"]
@ -46,11 +49,13 @@ class DockerRunner(Runner):
milvus.delete()
time.sleep(10)
milvus.create_table(table_name, dimension, index_file_size, metric_type)
# debug
# milvus.create_index("ivf_sq8", 16384)
res = self.do_insert(milvus, table_name, data_type, dimension, table_size, param["ni_per"])
logger.info(res)
# wait for file merge
time.sleep(6 * (table_size / 500000))
time.sleep(table_size * dimension / 5000000)
# Clear up
utils.remove_container(container)
@ -80,9 +85,11 @@ class DockerRunner(Runner):
for nlist in nlists:
result = milvus.describe_index()
logger.info(result)
milvus.create_index(index_type, nlist)
# milvus.drop_index()
# milvus.create_index(index_type, nlist)
result = milvus.describe_index()
logger.info(result)
logger.info(milvus.count())
# preload index
milvus.preload_table()
logger.info("Start warm up query")
@ -92,7 +99,7 @@ class DockerRunner(Runner):
for nprobe in nprobes:
logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count)
headers = ["Nprobe/Top-k"]
headers = ["Nq/Top-k"]
headers.extend([str(top_k) for top_k in top_ks])
utils.print_table(headers, nqs, res)
utils.remove_container(container)
@ -172,7 +179,18 @@ class DockerRunner(Runner):
avg_acc = self.compute_accuracy(base_name, id_prefix)
logger.info("Query: <%s> accuracy: %s" % (id_prefix, avg_acc))
else:
result_ids = self.do_query_ids(milvus, table_name, top_k, nq, nprobe)
result_ids, result_distances = self.do_query_ids(milvus, table_name, top_k, nq, nprobe)
debug_file_ids = "0.5.3_result_ids"
debug_file_distances = "0.5.3_result_distances"
with open(debug_file_ids, "w+") as fd:
total = 0
for index, item in enumerate(result_ids):
true_item = true_ids_all[:nq, :top_k].tolist()[index]
tmp = set(item).intersection(set(true_item))
total = total + len(tmp)
fd.write("query: N-%d, intersection: %d, total: %d\n" % (index, len(tmp), total))
fd.write("%s\n" % str(item))
fd.write("%s\n" % str(true_item))
acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids)
logger.info("Query: <%s> accuracy: %s" % (id_prefix, acc_value))
# # print accuracy table
@ -195,6 +213,7 @@ class DockerRunner(Runner):
for index, param in enumerate(run_params):
logger.info("Definition param: %s" % str(param))
table_name = param["dataset"]
index_type = param["index_type"]
volume_name = param["db_path_prefix"]
(data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
@ -223,7 +242,9 @@ class DockerRunner(Runner):
start_time = time.time()
insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(10000)]
i = 0
while time.time() < start_time + during_time:
i = i + 1
processes = []
# do query
# for i in range(query_process_num):
@ -243,7 +264,7 @@ class DockerRunner(Runner):
nqs = random.sample([x for x in range(1, 1000)], 3)
nprobe = random.choice([x for x in range(1, 500)])
res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count)
if int(time.time() - start_time) % 120 == 0:
if i % 10 == 0:
status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))])
if not status.OK():
logger.error(status)
@ -251,9 +272,10 @@ class DockerRunner(Runner):
# if not status.OK():
# logger.error(status)
# index_type = random.choice(["flat", "ivf_flat", "ivf_sq8"])
milvus_instance.create_index(index_type, 16384)
result = milvus.describe_index()
logger.info(result)
milvus_instance.create_index("ivf_sq8", 16384)
# milvus_instance.create_index("ivf_sq8", 16384)
utils.remove_container(container)
else:

119
tests/milvus_benchmark/local_runner.py
View File

@ -29,7 +29,6 @@ class LocalRunner(Runner):
if op_type == "insert":
for index, param in enumerate(run_params):
table_name = param["table_name"]
# random_1m_100_512
(data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
milvus = MilvusClient(table_name, ip=self.ip, port=self.port)
# Check has table or not
@ -47,25 +46,110 @@ class LocalRunner(Runner):
(data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
milvus = MilvusClient(table_name, ip=self.ip, port=self.port)
logger.info(milvus.describe())
logger.info(milvus.describe_index())
# parse index info
index_types = param["index.index_types"]
nlists = param["index.nlists"]
# parse top-k, nq, nprobe
top_ks, nqs, nprobes = parser.search_params_parser(param)
# milvus.drop_index()
for index_type in index_types:
for nlist in nlists:
milvus.create_index(index_type, nlist)
# milvus.create_index(index_type, nlist)
# preload index
logger.info("Start preloading table")
milvus.preload_table()
logger.info("End preloading table")
# Run query test
logger.info("Start warm up query")
res = self.do_query(milvus, table_name, [1], [1], 1, 1)
logger.info("End warm up query")
for nprobe in nprobes:
logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count)
headers = [param["dataset"]]
headers = ["nq/topk"]
headers.extend([str(top_k) for top_k in top_ks])
utils.print_table(headers, nqs, res)
elif run_type == "accuracy":
for op_type, op_value in definition.items():
if op_type != "query":
logger.warning("invalid operation: %s in accuracy test, only support query operation" % op_type)
break
run_count = op_value["run_count"]
run_params = op_value["params"]
for index, param in enumerate(run_params):
logger.info("Definition param: %s" % str(param))
table_name = param["dataset"]
sift_acc = False
if "sift_acc" in param:
sift_acc = param["sift_acc"]
(data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
milvus = MilvusClient(table_name, ip=self.ip, port=self.port)
logger.debug(milvus.show_tables())
# Check has table or not
if not milvus.exists_table():
logger.warning("Table %s not existed, continue exec next params ..." % table_name)
continue
# parse index info
index_types = param["index.index_types"]
nlists = param["index.nlists"]
# parse top-k, nq, nprobe
top_ks, nqs, nprobes = parser.search_params_parser(param)
if sift_acc is True:
# preload groundtruth data
true_ids_all = self.get_groundtruth_ids(table_size)
acc_dict = {}
for index_type in index_types:
for nlist in nlists:
result = milvus.describe_index()
logger.info(result)
# milvus.drop_index()
milvus.create_index(index_type, nlist)
# preload index
milvus.preload_table()
# Run query test
for nprobe in nprobes:
logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
for top_k in top_ks:
for nq in nqs:
result_ids = []
id_prefix = "%s_index_%s_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \
(table_name, index_type, nlist, metric_type, nprobe, top_k, nq)
if sift_acc is False:
self.do_query_acc(milvus, table_name, top_k, nq, nprobe, id_prefix)
if index_type != "flat":
# Compute accuracy
base_name = "%s_index_flat_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \
(table_name, nlist, metric_type, nprobe, top_k, nq)
avg_acc = self.compute_accuracy(base_name, id_prefix)
logger.info("Query: <%s> accuracy: %s" % (id_prefix, avg_acc))
else:
result_ids, result_distances = self.do_query_ids(milvus, table_name, top_k, nq, nprobe)
debug_file_ids = "0.5.3_result_ids"
debug_file_distances = "0.5.3_result_distances"
with open(debug_file_ids, "w+") as fd:
total = 0
for index, item in enumerate(result_ids):
true_item = true_ids_all[:nq, :top_k].tolist()[index]
tmp = set(item).intersection(set(true_item))
total = total + len(tmp)
fd.write("query: N-%d, intersection: %d, total: %d\n" % (index, len(tmp), total))
fd.write("%s\n" % str(item))
fd.write("%s\n" % str(true_item))
acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids)
logger.info("Query: <%s> accuracy: %s" % (id_prefix, acc_value))
# # print accuracy table
# headers = [table_name]
# headers.extend([str(top_k) for top_k in top_ks])
# utils.print_table(headers, nqs, res)
elif run_type == "stability":
for op_type, op_value in definition.items():
if op_type != "query":
@ -73,11 +157,12 @@ class LocalRunner(Runner):
break
run_count = op_value["run_count"]
run_params = op_value["params"]
nq = 10000
nq = 100000
for index, param in enumerate(run_params):
logger.info("Definition param: %s" % str(param))
table_name = param["dataset"]
index_type = param["index_type"]
(data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
# set default test time
@ -90,7 +175,10 @@ class LocalRunner(Runner):
query_process_num = 10
else:
query_process_num = int(param["query_process_num"])
milvus = MilvusClient(table_name)
milvus = MilvusClient(table_name, ip=self.ip, port=self.port)
logger.debug(milvus.show_tables())
logger.debug(milvus.describe_index())
logger.debug(milvus.count())
# Check has table or not
if not milvus.exists_table():
logger.warning("Table %s not existed, continue exec next params ..." % table_name)
@ -98,8 +186,9 @@ class LocalRunner(Runner):
start_time = time.time()
insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)]
i = 0
while time.time() < start_time + during_time:
processes = []
# processes = []
# # do query
# for i in range(query_process_num):
# milvus_instance = MilvusClient(table_name)
@ -113,20 +202,14 @@ class LocalRunner(Runner):
# time.sleep(0.1)
# for p in processes:
# p.join()
milvus_instance = MilvusClient(table_name)
i = i + 1
milvus_instance = MilvusClient(table_name, ip=self.ip, port=self.port)
top_ks = random.sample([x for x in range(1, 100)], 4)
nqs = random.sample([x for x in range(1, 1000)], 3)
nprobe = random.choice([x for x in range(1, 500)])
res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count)
nqs = random.sample([x for x in range(1, 200)], 3)
nprobe = random.choice([x for x in range(1, 100)])
res = self.do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count)
# milvus_instance = MilvusClient(table_name)
status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))])
if not status.OK():
logger.error(status.message)
if (time.time() - start_time) % 300 == 0:
status = milvus_instance.drop_index()
if not status.OK():
logger.error(status.message)
index_type = random.choice(["flat", "ivf_flat", "ivf_sq8"])
status = milvus_instance.create_index(index_type, 16384)
if not status.OK():
logger.error(status.message)
# status = milvus_instance.create_index(index_type, 16384)

11
tests/milvus_benchmark/requirements.txt
View File

@ -1,6 +1,9 @@
numpy==1.16.3
pymilvus>=0.1.18
pyyaml==5.1
docker==4.0.2
pymilvus-test>=0.2.0
scikit-learn==0.19.1
h5py==2.7.1
influxdb==5.2.2
pyyaml>=5.1
tableprint==0.8.0
ansicolors==1.1.8
ansicolors==1.1.8
scipy==1.3.1

52
tests/milvus_benchmark/runner.py
View File

@ -18,15 +18,18 @@ SIFT_VECTORS_PER_FILE = 100000
MAX_NQ = 10001
FILE_PREFIX = "binary_"
RANDOM_SRC_BINARY_DATA_DIR = '/tmp/random/binary_data'
SIFT_SRC_DATA_DIR = '/tmp/sift1b/query'
SIFT_SRC_BINARY_DATA_DIR = '/tmp/sift1b/binary_data'
SIFT_SRC_GROUNDTRUTH_DATA_DIR = '/tmp/sift1b/groundtruth'
# FOLDER_NAME = 'ann_1000m/source_data'
SRC_BINARY_DATA_DIR = '/poc/yuncong/yunfeng/random_data'
SRC_BINARY_DATA_DIR_high = '/test/milvus/raw_data/random'
SIFT_SRC_DATA_DIR = '/poc/yuncong/ann_1000m/'
SIFT_SRC_BINARY_DATA_DIR = SIFT_SRC_DATA_DIR + 'source_data'
SIFT_SRC_GROUNDTRUTH_DATA_DIR = SIFT_SRC_DATA_DIR + 'gnd'
WARM_TOP_K = 1
WARM_NQ = 1
DEFAULT_DIM = 512
GROUNDTRUTH_MAP = {
"1000000": "idx_1M.ivecs",
"2000000": "idx_2M.ivecs",
@ -45,7 +48,10 @@ def gen_file_name(idx, table_dimension, data_type):
s = "%05d" % idx
fname = FILE_PREFIX + str(table_dimension) + "d_" + s + ".npy"
if data_type == "random":
fname = RANDOM_SRC_BINARY_DATA_DIR+'/'+fname
if table_dimension == 512:
fname = SRC_BINARY_DATA_DIR+'/'+fname
elif table_dimension >= 4096:
fname = SRC_BINARY_DATA_DIR_high+'/'+fname
elif data_type == "sift":
fname = SIFT_SRC_BINARY_DATA_DIR+'/'+fname
return fname
@ -87,7 +93,12 @@ class Runner(object):
qps = 0.0
ni_time = 0.0
if data_type == "random":
vectors_per_file = VECTORS_PER_FILE
if dimension == 512:
vectors_per_file = VECTORS_PER_FILE
elif dimension == 4096:
vectors_per_file = 100000
elif dimension == 16384:
vectors_per_file = 10000
elif data_type == "sift":
vectors_per_file = SIFT_VECTORS_PER_FILE
if size % vectors_per_file or ni > vectors_per_file:
@ -101,15 +112,16 @@ class Runner(object):
loops = vectors_per_file // ni
for j in range(loops):
vectors = data[j*ni:(j+1)*ni].tolist()
ni_start_time = time.time()
# start insert vectors
start_id = i * vectors_per_file + j * ni
end_id = start_id + len(vectors)
logger.info("Start id: %s, end id: %s" % (start_id, end_id))
ids = [k for k in range(start_id, end_id)]
status, ids = milvus.insert(vectors, ids=ids)
ni_end_time = time.time()
total_time = total_time + ni_end_time - ni_start_time
if vectors:
ni_start_time = time.time()
# start insert vectors
start_id = i * vectors_per_file + j * ni
end_id = start_id + len(vectors)
logger.info("Start id: %s, end id: %s" % (start_id, end_id))
ids = [k for k in range(start_id, end_id)]
status, ids = milvus.insert(vectors, ids=ids)
ni_end_time = time.time()
total_time = total_time + ni_end_time - ni_start_time
qps = round(size / total_time, 2)
ni_time = round(total_time / (loops * file_num), 2)
@ -125,10 +137,10 @@ class Runner(object):
bi_res = []
for index, nq in enumerate(nqs):
tmp_res = []
vectors = base_query_vectors[0:nq]
for top_k in top_ks:
avg_query_time = 0.0
total_query_time = 0.0
vectors = base_query_vectors[0:nq]
logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors)))
for i in range(run_count):
logger.info("Start run query, run %d of %s" % (i+1, run_count))
@ -153,12 +165,16 @@ class Runner(object):
msg = "Query failed with message: %s" % status.message
raise Exception(msg)
result_ids = []
result_distances = []
for result in query_res:
tmp = []
tmp_distance = []
for item in result:
tmp.append(item.id)
tmp_distance.append(item.distance)
result_ids.append(tmp)
return result_ids
result_distances.append(tmp_distance)
return result_ids, result_distances
def do_query_acc(self, milvus, table_name, top_k, nq, nprobe, id_store_name):
(data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
@ -216,4 +232,4 @@ class Runner(object):
a = np.fromfile(fname, dtype='int32')
d = a[0]
true_ids = a.reshape(-1, d + 1)[:, 1:].copy()
return true_ids
return true_ids

57
tests/milvus_benchmark/suites/accuracy_random50m.yaml Normal file
View File

@ -0,0 +1,57 @@
accuracy:
# interface: search_vectors
query:
[
{
"dataset": "random_50m_1024_512_ip",
"index.index_types": ["flat", "ivf_flat"],
"index.nlists": [16384],
"index.metric_types": ["ip"],
"nprobes": [8, 64, 128, 512, 2048],
"top_ks": [64],
"nqs": [1000],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 100,
"server.cpu_cache_capacity": 120,
"server.gpu_cache_capacity": 6,
"server.enable_gpu": True,
"server.resources": ["gpu0", "gpu1"],
"db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_ivf",
"sift_acc": false
},
{
"dataset": "random_50m_1024_512_ip",
"index.index_types": ["flat", "ivf_sq8"],
"index.nlists": [16384],
"index.metric_types": ["ip"],
"nprobes": [8, 64, 128, 512, 2048],
"top_ks": [64],
"nqs": [1000],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 100,
"server.cpu_cache_capacity": 120,
"server.gpu_cache_capacity": 6,
"server.enable_gpu": True,
"server.resources": ["gpu0", "gpu1"],
"db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_sq8",
"sift_acc": false
},
{
"dataset": "random_50m_1024_512_ip",
"index.index_types": ["flat", "ivf_sq8h"],
"index.nlists": [16384],
"index.metric_types": ["ip"],
"nprobes": [8, 64, 128, 512, 2048],
"top_ks": [64],
"nqs": [1000],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 100,
"server.cpu_cache_capacity": 120,
"server.gpu_cache_capacity": 6,
"server.enable_gpu": True,
"server.resources": ["gpu0", "gpu1"],
"db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip",
"sift_acc": false
},
]

75
tests/milvus_benchmark/suites/accuracy_sift10m.yaml Normal file
View File

@ -0,0 +1,75 @@
accuracy:
# interface: search_vectors
query:
# dataset: table name you have already created
# key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
[
# {
# "dataset": "sift_10m_1024_128_l2",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "index.metric_types": ["l2"],
# "nprobes": [1, 8, 16, 32, 64, 128, 256],
# "top_ks": [64],
# "nqs": [1000],
# "server.cpu_cache_capacity": 50,
# "server.use_gpu_threshold": 1001,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["cpu", "gpu0"],
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8h",
# "sift_acc": true
# },
# {
# "dataset": "sift_10m_1024_128_l2",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "index.metric_types": ["l2"],
# "nprobes": [1, 8, 16, 32, 64, 128, 256],
# "top_ks": [64],
# "nqs": [1000],
# "server.cpu_cache_capacity": 50,
# "server.resources": ["cpu", "gpu0"],
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8",
# "sift_acc": true
# },
{
"dataset": "sift_10m_1024_128_l2",
"index.index_types": ["ivf_pq"],
"index.nlists": [16384],
"index.metric_types": ["l2"],
"nprobes": [1, 8, 16, 32, 64, 128, 256],
"top_ks": [64],
"nqs": [1000],
"server.cpu_cache_capacity": 50,
"server.resources": ["cpu", "gpu0"],
"db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8",
"sift_acc": true
},
# {
# "dataset": "sift_10m_1024_128_l2",
# "index.index_types": ["ivf_flat"],
# "index.nlists": [16384],
# "index.metric_types": ["l2"],
# "nprobes": [1, 8, 16, 32, 64, 128, 256],
# "top_ks": [64],
# "nqs": [1000],
# "server.cpu_cache_capacity": 100,
# "server.resources": ["cpu", "gpu0"],
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_ivf",
# "sift_acc": true
# },
# {
# "dataset": "sift_10m_1024_128_l2",
# "index.index_types": ["nsg"],
# "index.nlists": [16384],
# "index.metric_types": ["l2"],
# "nprobes": [1],
# "top_ks": [64],
# "nqs": [1000],
# # "server.cpu_cache_capacity": 100,
# # "server.resources": ["cpu", "gpu0"],
# "db_path_prefix": "/test/milvus/db_data_gpu/sift_10m_1024_128_l2_nsg",
# "sift_acc": true
# },
]

73
tests/milvus_benchmark/suites/accuracy_sift1b.yaml Normal file
View File

@ -0,0 +1,73 @@
accuracy:
# interface: search_vectors
query:
[
# {
# "dataset": "sift_1b_2048_128_l2",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "index.metric_types": ["l2"],
# # "nprobes": [1, 8, 32, 64, 128],
# "nprobes": [1, 8, 32, 64, 128, 256],
# "top_ks": [64],
# "nqs": [1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 1001,
# "server.cpu_cache_capacity": 150,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data_gpu/sift_1b_2048_128_l2",
# "sift_acc": true
# },
{
"dataset": "sift_1b_2048_128_l2",
"index.index_types": ["ivf_pq"],
"index.nlists": [16384],
"index.metric_types": ["l2"],
"nprobes": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
"top_ks": [64],
"nqs": [1000],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 10001,
"server.cpu_cache_capacity": 150,
"server.gpu_cache_capacity": 6,
"server.resources": ["gpu0", "gpu1"],
"db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_pq",
"sift_acc": true
},
# {
# "dataset": "sift_1b_2048_128_l2",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "index.metric_types": ["l2"],
# "nprobes": [1, 8, 32, 64, 128],
# "top_ks": [64],
# "nqs": [1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 150,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2",
# "sift_acc": true
# },
# {
# "dataset": "sift_1b_2048_128_l2",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "index.metric_types": ["l2"],
# "nprobes": [1, 8, 32, 64, 128],
# "top_ks": [64],
# "nqs": [1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 150,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["cpu", "gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2",
# "sift_acc": true
# },
]

21
tests/milvus_benchmark/suites/accuracy_sift500m.yaml Normal file
View File

@ -0,0 +1,21 @@
accuracy:
# interface: search_vectors
query:
# dataset: table name you have already created
# key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
[
{
"dataset": "sift_500m_1024_128_l2",
"index.index_types": ["ivf_sq8h"],
"index.nlists": [16384],
"index.metric_types": ["l2"],
"nprobes": [1, 8, 16, 32, 64, 128, 256],
"top_ks": [64],
"nqs": [1000],
"server.cpu_cache_capacity": 200,
"server.resources": ["cpu", "gpu0"],
"db_path_prefix": "/test/milvus/db_data/sift_500m_1024_128_l2",
"sift_acc": true
},
]

76
tests/milvus_benchmark/suites/accuracy_sift50m.yaml Normal file
View File

@ -0,0 +1,76 @@
accuracy:
# interface: search_vectors
query:
# dataset: table name you have already created
# key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
[
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "index.metric_types": ["l2"],
# "nprobes": [1, 8, 16, 32, 64, 128, 256],
# "top_ks": [64],
# "nqs": [1000],
# "server.cpu_cache_capacity": 50,
# "server.use_gpu_threshold": 1001,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["cpu", "gpu0"],
# "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8h",
# "sift_acc": true
# },
{
"dataset": "sift_50m_1024_128_l2",
"index.index_types": ["ivf_sq8"],
"index.nlists": [16384],
"index.metric_types": ["l2"],
"nprobes": [1, 8, 16, 32, 64, 128, 256],
"top_ks": [64],
"nqs": [1000],
"server.cpu_cache_capacity": 50,
"server.resources": ["cpu", "gpu0"],
"db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8",
"sift_acc": true
},
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["ivf_flat"],
# "index.nlists": [16384],
# "index.metric_types": ["l2"],
# "nprobes": [1, 8, 16, 32, 64, 128, 256],
# "top_ks": [64],
# "nqs": [1000],
# "server.cpu_cache_capacity": 100,
# "server.resources": ["cpu", "gpu0"],
# "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_ivf",
# "sift_acc": true
# },
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["ivf_flat"],
# "index.nlists": [16384],
# "index.metric_types": ["l2"],
# "nprobes": [1, 8, 16, 32, 64, 128, 256],
# "top_ks": [64],
# "nqs": [1000],
# "server.cpu_cache_capacity": 100,
# "server.resources": ["cpu", "gpu0"],
# "db_path_prefix": "/test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf",
# "sift_acc": true
# },
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["nsg"],
# "index.nlists": [16384],
# "index.metric_types": ["l2"],
# "nprobes": [1],
# "top_ks": [64],
# "nqs": [1000],
# "server.cpu_cache_capacity": 100,
# "server.resources": ["gpu0"],
# "db_path_prefix": "/test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg",
# "sift_acc": true
# },
]

32
tests/milvus_benchmark/suites/insert.yaml Normal file
View File

@ -0,0 +1,32 @@
performance:
# interface: add_vectors
insert:
# index_type: flat/ivf_flat/ivf_sq8/mix_nsg
[
# debug
# data_type / data_size / index_file_size / dimension
# data_type: random / ann_sift
# data_size: 10m / 1b
# {
# "table_name": "random_1m_1024_4096_ip",
# "ni_per": 10000,
# "server.cpu_cache_capacity": 16,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_1m_1024_4096_ip"
# },
# {
# "table_name": "random_1m_1024_16384_ip",
# "ni_per": 10000,
# "server.cpu_cache_capacity": 16,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_1m_1024_16384_ip"
# },
{
"table_name": "random_50m_1024_512_ip",
"ni_per": 100000,
# "server.cpu_cache_capacity": 16,
# "server.resources": ["gpu0", "gpu1"],
"db_path_prefix": "/test/milvus/db_data_cpu/random_50m_1024_512_ip_ivf"
},
]

80
tests/milvus_benchmark/suites/old_performance_sift50m.yaml Normal file
View File

@ -0,0 +1,80 @@
performance:
# interface: search_vectors
query:
# dataset: table name you have already created
# key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
[
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["mix_nsg"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_nsg"
# },
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["ivf_flat"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_ivf"
# },
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.gpu_cache_capacity": 3,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8"
# },
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8"
# },
{
"dataset": "sift_50m_1024_128_l2",
"index.index_types": ["ivf_sq8"],
"index.nlists": [16384],
"nprobes": [8, 32],
"top_ks": [1, 16, 64, 128, 256, 512, 1000],
"nqs": [1, 10, 100, 200, 500, 1000],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 100,
"server.cpu_cache_capacity": 50,
"server.gpu_cache_capacity": 6,
"server.resources": ["gpu0", "gpu1"],
"db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8"
},
]

35
tests/milvus_benchmark/suites/performance_random1m.yaml Normal file
View File

@ -0,0 +1,35 @@
performance:
# interface: search_vectors
query:
# dataset: table name you have already created
# key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
[
# {
# "dataset": "random_1m_1024_4096_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 1100,
# "server.cpu_cache_capacity": 50,
# "server.resources": ["cpu", "gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_1m_1024_4096_ip_sq8h"
# },
{
"dataset": "random_2m_1024_512_ip",
"index.index_types": ["mix_nsg"],
"index.nlists": [16384],
"nprobes": [8],
"top_ks": [1],
"nqs": [1],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 1100,
"server.cpu_cache_capacity": 50,
"server.resources": ["gpu0", "gpu1"],
"server.enable_gpu": True,
"db_path_prefix": "/test/milvus/db_data/random_1m_1024_4096_ip_sq8"
},
]

177
tests/milvus_benchmark/suites/performance_random50m.yaml Normal file
View File

@ -0,0 +1,177 @@
performance:
# interface: search_vectors
query:
# dataset: table name you have already created
# key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
[
# {
# "dataset": "random_50m_2048_512_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.resources": ["cpu", "gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h"
# },
# {
# "dataset": "random_50m_2048_512_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 1000,
# "server.cpu_cache_capacity": 50,
# "server.resources": ["cpu", "gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h"
# },
# {
# "dataset": "random_50m_2048_512_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 10000,
# "server.cpu_cache_capacity": 50,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h"
# },
# {
# "dataset": "random_50m_2048_512_ip",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8"
# },
{
"dataset": "random_50m_2048_512_ip",
"index.index_types": ["ivf_sq8"],
"index.nlists": [16384],
"nprobes": [8],
"top_ks": [64],
"nqs": [1, 10, 100],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 100,
"server.cpu_cache_capacity": 50,
"server.enable_gpu": True,
"server.resources": ["gpu0", "gpu1"],
"db_path_prefix": "/test/milvus/db_data_cpu/random_50m_2048_512_ip_sq8"
},
# {
# "dataset": "random_50m_1024_512_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 110,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip"
# },
# {
# "dataset": "random_50m_1024_512_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 1000,
# "server.cpu_cache_capacity": 110,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip"
# },
# {
# "dataset": "random_50m_1024_512_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 10000,
# "server.cpu_cache_capacity": 110,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip"
# },
# {
# "dataset": "random_50m_1024_512_ip",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.cpu_cache_capacity": 110,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8"
# },
# {
# "dataset": "random_50m_1024_512_ip",
# "index.index_types": ["ivf_pq"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 110,
# "server.gpu_cache_capacity": 6,
# "server.index_build_device": "gpu0",
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_pq"
# },
# {
# "dataset": "random_50m_1024_512_ip",
# "index.index_types": ["ivf_flat"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 110,
# "server.gpu_cache_capacity": 6,
# "server.index_build_device": "gpu0",
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_ivf"
# },
# {
# "dataset": "random_50m_1024_512_ip",
# "index.index_types": ["nsg"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 110,
# "server.gpu_cache_capacity": 6,
# "server.index_build_device": "gpu0",
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_nsg"
# }
]

83
tests/milvus_benchmark/suites/performance_sift10m.yaml Normal file
View File

@ -0,0 +1,83 @@
performance:
# interface: search_vectors
query:
# dataset: table name you have already created
# key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
[
# {
# "dataset": "sift_10m_1024_128_l2",
# "index.index_types": ["ivf_flat"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0"],
# "server.enable_gpu": False,
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_ivf"
# },
# {
# "dataset": "sift_10m_1024_128_l2",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0"],
# "server.enable_gpu": False,
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_sq8"
# },
# {
# "dataset": "sift_10m_1024_128_l2",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0"],
# "server.enable_gpu": False,
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_sq8h"
# },
{
"dataset": "sift_10m_1024_128_l2",
"index.index_types": ["ivf_pq"],
"index.nlists": [16384],
"nprobes": [8, 32],
"top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
"nqs": [1, 10, 100, 500, 800, 1000],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 100,
"server.cpu_cache_capacity": 50,
"server.gpu_cache_capacity": 6,
"server.resources": ["gpu0"],
"server.enable_gpu": False,
"db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_pq"
},
# {
# "dataset": "sift_10m_1024_128_l2",
# "index.index_types": ["nsg"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0"],
# "server.enable_gpu": False,
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_nsg"
# },
]

123
tests/milvus_benchmark/suites/performance_sift1b.yaml Normal file
View File

@ -0,0 +1,123 @@
performance:
# interface: search_vectors
query:
# dataset: table name you have already created
# key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
[
# {
# "dataset": "sift_1b_2048_128_l2",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 10000,
# "server.cpu_cache_capacity": 150,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h"
# },
# {
# "dataset": "sift_1b_2048_128_l2",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 150,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h"
# },
# {
# "dataset": "sift_1b_2048_128_l2",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 150,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["cpu", "gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h"
# },
# {
# "dataset": "sift_1b_2048_128_l2",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 1000,
# "server.cpu_cache_capacity": 150,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h"
# },
# {
# "dataset": "sift_1b_2048_128_l2",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 10000,
# "server.cpu_cache_capacity": 150,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": False,
# "db_path_prefix": "/test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h"
# },
{
"dataset": "sift_1b_2048_128_l2",
"index.index_types": ["ivf_pq"],
"index.nlists": [16384],
"nprobes": [8, 32],
"top_ks": [1, 16, 64, 128, 256, 512, 1000],
"nqs": [1, 10, 100, 200, 500, 1000],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 10000,
"server.cpu_cache_capacity": 150,
"server.gpu_cache_capacity": 6,
"server.resources": ["gpu0", "gpu1"],
"server.enable_gpu": False,
"db_path_prefix": "/test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq"
},
# {
# "dataset": "sift_1b_2048_128_l2",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 150,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": False,
# "db_path_prefix": "/test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8"
# },
# {
# "dataset": "sift_1b_2048_128_l2",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 150,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["cpu", "gpu0"],
# "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2"
# },
]

23
tests/milvus_benchmark/suites/performance_sift1m.yaml Normal file
View File

@ -0,0 +1,23 @@
performance:
# interface: search_vectors
query:
# dataset: table name you have already created
# key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
[
{
"dataset": "sift_1m_1024_128_l2",
"index.index_types": ["mix_nsg"],
"index.nlists": [16384],
"nprobes": [8, 32],
"top_ks": [1, 16, 64, 128, 256, 512, 1000],
"nqs": [1, 10, 100, 200, 500, 1000],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 100,
"server.cpu_cache_capacity": 50,
"server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
"server.enable_gpu": True,
"db_path_prefix": "/test/milvus/db_data/sift_1m_1024_128_l2_nsg"
},
]

50
tests/milvus_benchmark/suites/performance_sift500m.yaml Normal file
View File

@ -0,0 +1,50 @@
performance:
# interface: search_vectors
query:
# dataset: table name you have already created
# key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
[
# {
# "dataset": "sift_500m_1024_128_l2",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 150,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/sift_500m_1024_128_l2_sq8"
# },
# {
# "dataset": "sift_500m_1024_128_l2",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 150,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/sift_500m_1024_128_l2_sq8"
# },
{
"dataset": "sift_500m_1024_128_l2",
"index.index_types": ["ivf_sq8h"],
"index.nlists": [16384],
"nprobes": [8, 32],
"top_ks": [1, 100, 200, 1000],
"nqs": [1, 10, 100, 500, 1000],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 100,
"server.cpu_cache_capacity": 150,
"server.gpu_cache_capacity": 6,
"server.resources": ["gpu0", "gpu1"],
"db_path_prefix": "/test/milvus/db_data/sift_500m_1024_128_l2"
},
]

82
tests/milvus_benchmark/suites/performance_sift50m.yaml Normal file
View File

@ -0,0 +1,82 @@
performance:
# interface: search_vectors
query:
# dataset: table name you have already created
# key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
[
{
"dataset": "sift_50m_1024_128_l2",
"index.index_types": ["nsg"],
"index.nlists": [16384],
"nprobes": [8],
"top_ks": [64],
"nqs": [1, 10, 100],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 100,
"server.cpu_cache_capacity": 50,
"server.gpu_cache_capacity": 6,
"server.resources": ["gpu0", "gpu1"],
"server.enable_gpu": True,
"db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_nsg"
},
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["ivf_flat"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_ivf"
# },
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.gpu_cache_capacity": 3,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h"
# },
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8"
# },
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["ivf_pq"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 200, 500, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.gpu_cache_capacity": 6,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_pq"
# },
]

161
tests/milvus_benchmark/suites/suites.yaml Normal file
View File

@ -0,0 +1,161 @@
performance:
# interface: search_vectors
query:
# dataset: table name you have already created
# key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
[
# {
# "dataset": "random_50m_2048_512_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.resources": ["cpu", "gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h"
# },
# {
# "dataset": "random_50m_2048_512_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 1000,
# "server.cpu_cache_capacity": 50,
# "server.resources": ["cpu", "gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h"
# },
# {
# "dataset": "random_50m_2048_512_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 10000,
# "server.cpu_cache_capacity": 50,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h"
# },
# {
# "dataset": "random_50m_2048_512_ip",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8"
# },
# {
# "dataset": "random_50m_2048_512_ip",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 50,
# "server.enable_gpu": True,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data_cpu/random_50m_2048_512_ip_sq8"
# },
# {
# "dataset": "random_50m_1024_512_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 110,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip"
# },
# {
# "dataset": "random_50m_1024_512_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 1000,
# "server.cpu_cache_capacity": 110,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip"
# },
# {
# "dataset": "random_50m_1024_512_ip",
# "index.index_types": ["ivf_sq8h"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 10000,
# "server.cpu_cache_capacity": 110,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip"
# },
# {
# "dataset": "random_50m_1024_512_ip",
# "index.index_types": ["ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [8, 32],
# "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
# "nqs": [1, 10, 100, 500, 800, 1000],
# "server.use_blas_threshold": 1100,
# "server.cpu_cache_capacity": 110,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8"
# },
{
"dataset": "random_50m_1024_512_ip",
"index.index_types": ["ivf_pq"],
"index.nlists": [16384],
"nprobes": [8, 32],
"top_ks": [1, 4, 16, 64, 128, 256, 512, 1000],
"nqs": [1, 10, 100, 500, 800, 1000],
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 100,
"server.cpu_cache_capacity": 110,
"server.gpu_cache_capacity": 6,
"server.index_build_device": "gpu0",
"server.resources": ["gpu0", "gpu1"],
"server.enable_gpu": True,
"db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_pq"
},
# {
# "dataset": "random_50m_1024_512_ip",
# "index.index_types": ["ivf_flat"],
# "index.nlists": [16384],
# "nprobes": [8],
# "top_ks": [8],
# "nqs": [500],
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.cpu_cache_capacity": 110,
# "server.gpu_cache_capacity": 6,
# "server.index_build_device": "gpu0",
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": True,
# "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_ivf"
# }
]

51
tests/milvus_benchmark/suites/suites_accuracy.yaml Normal file
View File

@ -0,0 +1,51 @@
accuracy:
# interface: search_vectors
query:
[
# {
# "dataset": "random_20m_1024_512_ip",
# # index info
# "index.index_types": ["flat", "ivf_sq8"],
# "index.nlists": [16384],
# "index.metric_types": ["ip"],
# "nprobes": [1, 16, 64],
# "top_ks": [64],
# "nqs": [100],
# "server.cpu_cache_capacity": 100,
# "server.resources": ["gpu0"],
# "db_path_prefix": "/test/milvus/db_data/random_20m_1024_512_ip",
# },
# {
# "dataset": "sift_1m_1024_128_l2",
# "index.index_types": ["flat", "ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [1, 32, 128, 256, 512],
# "nqs": 10,
# "top_ks": 10,
# "server.use_blas_threshold": 1100,
# "server.cpu_cache_capacity": 16,
# },
# {
# "dataset": "sift_10m_1024_128_l2",
# "index.index_types": ["flat", "ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [1, 32, 128, 256, 512],
# "nqs": 10,
# "top_ks": 10,
# "server.use_blas_threshold": 1100,
# "server.cpu_cache_capacity": 32,
# },
# {
# "dataset": "sift_50m_1024_128_l2",
# "index.index_types": ["flat", "ivf_sq8"],
# "index.nlists": [16384],
# "nprobes": [1, 32, 128, 256, 512],
# "nqs": 10,
# "top_ks": 10,
# "server.use_blas_threshold": 1100,
# "server.cpu_cache_capacity": 64,
# }
]

57
tests/milvus_benchmark/suites/suites_performance.yaml Normal file
View File

@ -0,0 +1,57 @@
performance:
# interface: add_vectors
insert:
# index_type: flat/ivf_flat/ivf_sq8/mix_nsg
[
# debug
# data_type / data_size / index_file_size / dimension
# data_type: random / ann_sift
# data_size: 10m / 1b
# {
# "table_name": "random_50m_1024_512_ip",
# "ni_per": 100000,
# "server.cpu_cache_capacity": 16,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data"
# },
# {
# "table_name": "random_2m_1024_512_ip",
# "ni_per": 100000,
# "processes": 5, # multiprocessing
# "server.cpu_cache_capacity": 16,
# "server.resources": ["gpu0", "gpu1"],
# "db_path_prefix": "/test/milvus/db_data/random_5m_1024_512_ip"
# },
# {
# "table_name": "sift_1m_50_128_l2",
# "ni_per": 100000,
# "processes": 5, # multiprocessing
# # "server.cpu_cache_capacity": 16,
# "db_path_prefix": "/test/milvus/db_data"
# },
# {
# "table_name": "sift_1m_256_128_l2",
# "ni_per": 100000,
# "processes": 5, # multiprocessing
# # "server.cpu_cache_capacity": 16,
# "db_path_prefix": "/test/milvus/db_data"
# }
# {
# "table_name": "sift_50m_1024_128_l2",
# "ni_per": 100000,
# "processes": 5, # multiprocessing
# # "server.cpu_cache_capacity": 16,
# },
# {
# "table_name": "sift_100m_1024_128_l2",
# "ni_per": 100000,
# "processes": 5, # multiprocessing
# },
# {
# "table_name": "sift_1b_2048_128_l2",
# "ni_per": 100000,
# "processes": 5, # multiprocessing
# "server.cpu_cache_capacity": 16,
# }
]

31
tests/milvus_benchmark/suites/suites_stability.yaml Normal file
View File

@ -0,0 +1,31 @@
stability:
# interface: search_vectors / add_vectors mix operation
query:
[
# {
# "dataset": "random_50m_1024_512_ip",
# "index_type": "ivf_sq8",
# "query_process_num": 10,
# "during_time": 960,
# "server.cpu_cache_capacity": 100,
# "server.use_blas_threshold": 1100,
# "server.use_gpu_threshold": 100,
# "server.resources": ["gpu0", "gpu1"],
# "server.enable_gpu": false,
# "db_path_prefix": "/test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8_stability"
# },
{
"dataset": "random_50m_1024_512_ip",
"index_type": "ivf_sq8",
"query_process_num": 10,
"during_time": 960,
"server.cpu_cache_capacity": 100,
"server.use_blas_threshold": 1100,
"server.use_gpu_threshold": 100,
"server.resources": ["gpu0", "gpu1"],
"server.enable_gpu": false,
"db_path_prefix": "/test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8_stability"
},
]

14
tests/milvus_benchmark/utils.py
View File

@ -45,12 +45,18 @@ def modify_config(k, v, type=None, file_path="conf/server_config.yaml", db_slave
if config_dict:
if k.find("use_blas_threshold") != -1:
config_dict['engine_config']['use_blas_threshold'] = int(v)
elif k.find("use_gpu_threshold") != -1:
config_dict['engine_config']['gpu_search_threshold'] = int(v)
elif k.find("cpu_cache_capacity") != -1:
config_dict['cache_config']['cpu_cache_capacity'] = int(v)
elif k.find("enable_gpu") != -1:
config_dict['gpu_resource_config']['enable'] = v
elif k.find("gpu_cache_capacity") != -1:
config_dict['cache_config']['gpu_cache_capacity'] = int(v)
elif k.find("resource_pool") != -1:
config_dict['resource_config']['resource_pool'] = v
config_dict['gpu_resource_config']['cache_capacity'] = int(v)
elif k.find("index_build_device") != -1:
config_dict['gpu_resource_config']['build_index_resources'] = v
elif k.find("search_resources") != -1:
config_dict['resource_config']['resources'] = v
if db_slave:
config_dict['db_config']['db_slave_path'] = MULTI_DB_SLAVE_PATH
@ -123,7 +129,7 @@ def run_server(image, mem_limit=None, timeout=30, test_type="local", volume_name
volumes=volumes,
runtime="nvidia",
ports={'19530/tcp': 19530, '8080/tcp': 8080},
environment=["OMP_NUM_THREADS=48"],
# environment=["OMP_NUM_THREADS=48"],
# cpuset_cpus=cpu_limit,
# mem_limit=mem_limit,
# environment=[""],