From 5148fc150b2eb5c2709526e3a6bb07afadef4a9e Mon Sep 17 00:00:00 2001 From: "allcontributors[bot]" <46447321+allcontributors[bot]@users.noreply.github.com> Date: Wed, 4 Dec 2019 11:42:34 +0800 Subject: [PATCH 1/2] docs: add gaolizhou as a contributor (#679) * docs: update README.md [skip ci] * docs: create .all-contributorsrc [skip ci] --- .all-contributorsrc | 24 ++++++++++++++++++++++++ README.md | 13 +++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 .all-contributorsrc diff --git a/.all-contributorsrc b/.all-contributorsrc new file mode 100644 index 0000000000..9a1cd217b6 --- /dev/null +++ b/.all-contributorsrc @@ -0,0 +1,24 @@ +{ + "files": [ + "README.md" + ], + "imageSize": 100, + "commit": false, + "contributors": [ + { + "login": "gaolizhou", + "name": "gaolizhou", + "avatar_url": "https://avatars2.githubusercontent.com/u/2884044?v=4", + "profile": "https://github.com/gaolizhou", + "contributions": [ + "code" + ] + } + ], + "contributorsPerLine": 7, + "projectName": "milvus", + "projectOwner": "milvus-io", + "repoType": "github", + "repoHost": "https://github.com", + "skipCi": true +} diff --git a/README.md b/README.md index f975f0437e..214acac775 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ +[![All Contributors](https://img.shields.io/badge/all_contributors-1-orange.svg?style=flat-square)](#contributors-) ![Milvuslogo](https://github.com/milvus-io/docs/blob/master/assets/milvus_logo.png) [![Slack](https://img.shields.io/badge/Join-Slack-orange)](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk) @@ -56,6 +57,18 @@ To connect with other users and contributors, welcome to join our [Slack channel ## Contributors Below is a list of Milvus contributors. We greatly appreciate your contributions! + + + + + + + +
gaolizhou
gaolizhou
💻
+ + + + - [akihoni](https://github.com/akihoni) provided the CN version of README, and found a broken link in the doc. - [goodhamgupta](https://github.com/goodhamgupta) fixed a filename typo in the bootcamp doc. From beed46f79c6436350ca6bf9d279553309653b933 Mon Sep 17 00:00:00 2001 From: del-zhenwu <56623710+del-zhenwu@users.noreply.github.com> Date: Wed, 4 Dec 2019 14:18:20 +0800 Subject: [PATCH 2/2] Upgrade pyyaml and update tests (#681) --- tests/milvus_ann_acc/client.py | 6 +- tests/milvus_ann_acc/requirements.txt | 6 +- tests/milvus_ann_acc/runner.py | 6 +- tests/milvus_ann_acc/suite.yaml | 14 +- tests/milvus_ann_acc/suite_cpu.yaml | 19 ++ tests/milvus_ann_acc/suite_debug.yaml | 13 +- tests/milvus_ann_acc/suite_gpu.yaml | 19 ++ tests/milvus_benchmark/client.py | 127 ++++++++----- .../milvus_benchmark/conf/server_config.yaml | 60 +++--- tests/milvus_benchmark/docker_runner.py | 34 +++- tests/milvus_benchmark/local_runner.py | 119 ++++++++++-- tests/milvus_benchmark/requirements.txt | 11 +- tests/milvus_benchmark/runner.py | 52 +++-- .../suites/accuracy_random50m.yaml | 57 ++++++ .../suites/accuracy_sift10m.yaml | 75 ++++++++ .../suites/accuracy_sift1b.yaml | 73 ++++++++ .../suites/accuracy_sift500m.yaml | 21 +++ .../suites/accuracy_sift50m.yaml | 76 ++++++++ tests/milvus_benchmark/suites/insert.yaml | 32 ++++ .../suites/old_performance_sift50m.yaml | 80 ++++++++ .../suites/performance_random1m.yaml | 35 ++++ .../suites/performance_random50m.yaml | 177 ++++++++++++++++++ .../suites/performance_sift10m.yaml | 83 ++++++++ .../suites/performance_sift1b.yaml | 123 ++++++++++++ .../suites/performance_sift1m.yaml | 23 +++ .../suites/performance_sift500m.yaml | 50 +++++ .../suites/performance_sift50m.yaml | 82 ++++++++ tests/milvus_benchmark/suites/suites.yaml | 161 ++++++++++++++++ .../suites/suites_accuracy.yaml | 51 +++++ .../suites/suites_performance.yaml | 57 ++++++ .../suites/suites_stability.yaml | 31 +++ tests/milvus_benchmark/utils.py | 14 +- 32 files changed, 1649 insertions(+), 138 deletions(-) create mode 100644 tests/milvus_ann_acc/suite_cpu.yaml create mode 100644 tests/milvus_ann_acc/suite_gpu.yaml create mode 100644 tests/milvus_benchmark/suites/accuracy_random50m.yaml create mode 100644 tests/milvus_benchmark/suites/accuracy_sift10m.yaml create mode 100644 tests/milvus_benchmark/suites/accuracy_sift1b.yaml create mode 100644 tests/milvus_benchmark/suites/accuracy_sift500m.yaml create mode 100644 tests/milvus_benchmark/suites/accuracy_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/insert.yaml create mode 100644 tests/milvus_benchmark/suites/old_performance_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/performance_random1m.yaml create mode 100644 tests/milvus_benchmark/suites/performance_random50m.yaml create mode 100644 tests/milvus_benchmark/suites/performance_sift10m.yaml create mode 100644 tests/milvus_benchmark/suites/performance_sift1b.yaml create mode 100644 tests/milvus_benchmark/suites/performance_sift1m.yaml create mode 100644 tests/milvus_benchmark/suites/performance_sift500m.yaml create mode 100644 tests/milvus_benchmark/suites/performance_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/suites.yaml create mode 100644 tests/milvus_benchmark/suites/suites_accuracy.yaml create mode 100644 tests/milvus_benchmark/suites/suites_performance.yaml create mode 100644 tests/milvus_benchmark/suites/suites_stability.yaml diff --git a/tests/milvus_ann_acc/client.py b/tests/milvus_ann_acc/client.py index 6fec829612..4ec3cb151a 100644 --- a/tests/milvus_ann_acc/client.py +++ b/tests/milvus_ann_acc/client.py @@ -88,8 +88,10 @@ class MilvusClient(object): index_type = IndexType.IVF_SQ8 elif index_type == "ivf_sq8h": index_type = IndexType.IVF_SQ8H - elif index_type == "mix_nsg": - index_type = IndexType.MIX_NSG + elif index_type == "nsg": + index_type = IndexType.NSG + elif index_type == "ivf_pq": + index_type = IndexType.IVF_PQ index_params = { "index_type": index_type, "nlist": nlist, diff --git a/tests/milvus_ann_acc/requirements.txt b/tests/milvus_ann_acc/requirements.txt index 1f2b337423..ee6a8a11ff 100644 --- a/tests/milvus_ann_acc/requirements.txt +++ b/tests/milvus_ann_acc/requirements.txt @@ -1,9 +1,9 @@ numpy==1.16.3 -pymilvus>=0.2.0 +pymilvus-test>=0.2.0 scikit-learn==0.19.1 h5py==2.7.1 influxdb==5.2.2 -pyyaml==3.12 +pyyaml>=5.1 tableprint==0.8.0 ansicolors==1.1.8 -scipy==1.3.1 \ No newline at end of file +scipy==1.3.1 diff --git a/tests/milvus_ann_acc/runner.py b/tests/milvus_ann_acc/runner.py index 88a5d24016..ac5fc0a6dc 100644 --- a/tests/milvus_ann_acc/runner.py +++ b/tests/milvus_ann_acc/runner.py @@ -55,9 +55,9 @@ def recall_calc(result_ids, true_ids, top_k, recall_k): sum_intersect_num = 0 recall = 0.0 for index, result_item in enumerate(result_ids): - if len(set(true_ids[index][:top_k])) != len(set(result_item)): - logger.warning("Error happened: query result length is wrong") - continue + # if len(set(true_ids[index][:top_k])) != len(set(result_item)): + # logger.warning("Error happened: query result length is wrong") + # continue tmp = set(true_ids[index][:recall_k]).intersection(set(result_item)) sum_intersect_num = sum_intersect_num + len(tmp) recall = round(sum_intersect_num / (len(result_ids) * recall_k), 4) diff --git a/tests/milvus_ann_acc/suite.yaml b/tests/milvus_ann_acc/suite.yaml index 1137ccfa64..7e772649a6 100644 --- a/tests/milvus_ann_acc/suite.yaml +++ b/tests/milvus_ann_acc/suite.yaml @@ -1,29 +1,29 @@ datasets: - sift-128-euclidean: - index_file_sizes: [50, 1024] + index_file_sizes: [1024] index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h'] # index_types: ['ivf_sq8'] nlists: [16384] search_param: - nprobes: [1, 32, 128, 256] + nprobes: [1, 8, 16, 32, 64, 128, 256, 512] top_ks: [10] nqs: [10000] - glove-25-angular: - index_file_sizes: [50, 1024] + index_file_sizes: [1024] index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h'] # index_types: ['ivf_sq8'] nlists: [16384] search_param: - nprobes: [1, 32, 128, 256] + nprobes: [1, 8, 16, 32, 64, 128, 256, 512] top_ks: [10] nqs: [10000] - glove-200-angular: - index_file_sizes: [50, 1024] + index_file_sizes: [1024] index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h'] # index_types: ['ivf_sq8'] nlists: [16384] search_param: - nprobes: [1, 32, 128, 256] + nprobes: [1, 8, 16, 32, 64, 128, 256, 512] top_ks: [10] nqs: [10000] -hdf5_path: /test/milvus/ann_hdf5/ \ No newline at end of file +hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_ann_acc/suite_cpu.yaml b/tests/milvus_ann_acc/suite_cpu.yaml new file mode 100644 index 0000000000..44caaeb91c --- /dev/null +++ b/tests/milvus_ann_acc/suite_cpu.yaml @@ -0,0 +1,19 @@ +datasets: + - sift-128-euclidean: + index_file_sizes: [1024] + index_types: ['ivf_flat', 'ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] + - glove-200-angular: + index_file_sizes: [1024] + index_types: ['ivf_flat', 'ivf_sq8'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] +hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_ann_acc/suite_debug.yaml b/tests/milvus_ann_acc/suite_debug.yaml index ca463a9c40..44caaeb91c 100644 --- a/tests/milvus_ann_acc/suite_debug.yaml +++ b/tests/milvus_ann_acc/suite_debug.yaml @@ -1,10 +1,19 @@ datasets: - sift-128-euclidean: index_file_sizes: [1024] - index_types: ['ivf_flat'] + index_types: ['ivf_flat', 'ivf_sq8'] nlists: [16384] search_param: - nprobes: [1, 256] + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] + - glove-200-angular: + index_file_sizes: [1024] + index_types: ['ivf_flat', 'ivf_sq8'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] top_ks: [10] nqs: [10000] hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_ann_acc/suite_gpu.yaml b/tests/milvus_ann_acc/suite_gpu.yaml new file mode 100644 index 0000000000..e83433ffb9 --- /dev/null +++ b/tests/milvus_ann_acc/suite_gpu.yaml @@ -0,0 +1,19 @@ +datasets: + - sift-128-euclidean: + index_file_sizes: [1024] + index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'ivf_pq'] + nlists: [16384] + search_param: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] + - glove-200-angular: + index_file_sizes: [1024] + index_types: ['ivf_flat', 'ivf_sq8'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] +hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_benchmark/client.py b/tests/milvus_benchmark/client.py index 4744c10854..608d760813 100644 --- a/tests/milvus_benchmark/client.py +++ b/tests/milvus_benchmark/client.py @@ -10,6 +10,7 @@ from milvus import Milvus, IndexType, MetricType logger = logging.getLogger("milvus_benchmark.client") SERVER_HOST_DEFAULT = "127.0.0.1" +# SERVER_HOST_DEFAULT = "192.168.1.130" SERVER_PORT_DEFAULT = 19530 @@ -80,16 +81,18 @@ class MilvusClient(object): index_type = IndexType.IVFLAT elif index_type == "ivf_sq8": index_type = IndexType.IVF_SQ8 - elif index_type == "mix_nsg": - index_type = IndexType.MIX_NSG + elif index_type == "nsg": + index_type = IndexType.NSG elif index_type == "ivf_sq8h": index_type = IndexType.IVF_SQ8H + elif index_type == "ivf_pq": + index_type = IndexType.IVF_PQ index_params = { "index_type": index_type, "nlist": nlist, } logger.info("Building index start, table_name: %s, index_params: %s" % (self._table_name, json.dumps(index_params))) - status = self._milvus.create_index(self._table_name, index=index_params, timeout=6*3600) + status = self._milvus.create_index(self._table_name, index=index_params) self.check_status(status) def describe_index(self): @@ -125,8 +128,13 @@ class MilvusClient(object): def describe(self): return self._milvus.describe_table(self._table_name) + def show_tables(self): + return self._milvus.show_tables() + def exists_table(self): - return self._milvus.has_table(self._table_name) + status, res = self._milvus.has_table(self._table_name) + self.check_status(status) + return res @time_wrapper def preload_table(self): @@ -154,34 +162,60 @@ def fit_concurrent(table_name, process_num, vectors): if __name__ == "__main__": + import numpy + import sklearn.preprocessing - # table_name = "sift_2m_20_128_l2" - table_name = "test_tset1" - m = MilvusClient(table_name) - # m.create_table(table_name, 128, 50, "l2") + # table_name = "tset_test" + # # table_name = "test_tset1" + # m = MilvusClient(table_name) + # m.delete() + # time.sleep(2) + # m.create_table(table_name, 128, 20, "ip") - print(m.describe()) + # print(m.describe()) # print(m.count()) # print(m.describe_index()) - insert_vectors = [[random.random() for _ in range(128)] for _ in range(10000)] - for i in range(5): - m.insert(insert_vectors) - print(m.create_index("ivf_sq8h", 16384)) - X = [insert_vectors[0]] - top_k = 10 - nprobe = 10 - print(m.query(X, top_k, nprobe)) - - # # # print(m.drop_index()) - # # print(m.describe_index()) # # sys.exit() - # # # insert_vectors = [[random.random() for _ in range(128)] for _ in range(100000)] - # # # for i in range(100): - # # # m.insert(insert_vectors) - # # # time.sleep(5) - # # # print(m.describe_index()) - # # # print(m.drop_index()) - # # m.create_index("ivf_sq8h", 16384) + # tmp = [[random.random() for _ in range(128)] for _ in range(20000)] + # tmp1 = sklearn.preprocessing.normalize(tmp, axis=1, norm='l2') + # print(tmp1[0][0]) + # tmp = [[random.random() for _ in range(128)] for _ in range(20000)] + # tmp /= numpy.linalg.norm(tmp) + # print(tmp[0][0]) + + # sum_1 = 0 + # sum_2 = 0 + # for item in tmp: + # for i in item: + # sum_2 = sum_2 + i * i + # break + # for item in tmp1: + # for i in item: + # sum_1 = sum_1 + i * i + # break + # print(sum_1, sum_2) + # insert_vectors = tmp.tolist() + # # print(insert_vectors) + # for i in range(2): + # m.insert(insert_vectors) + + # time.sleep(5) + # print(m.create_index("ivf_flat", 16384)) + # X = [insert_vectors[0], insert_vectors[1], insert_vectors[2]] + # top_k = 5 + # nprobe = 1 + # print(m.query(X, top_k, nprobe)) + + # # print(m.drop_index()) + # print(m.describe_index()) + # sys.exit() + # # insert_vectors = [[random.random() for _ in range(128)] for _ in range(100000)] + # # for i in range(100): + # # m.insert(insert_vectors) + # # time.sleep(5) + # # print(m.describe_index()) + # # print(m.drop_index()) + # m.create_index("ivf_sq8h", 16384) # print(m.count()) # print(m.describe_index()) @@ -193,12 +227,12 @@ if __name__ == "__main__": # print(m.describe_index()) import numpy as np - def mmap_fvecs(fname): - x = np.memmap(fname, dtype='int32', mode='r') - d = x[0] - return x.view('float32').reshape(-1, d + 1)[:, 1:] + # def mmap_fvecs(fname): + # x = np.memmap(fname, dtype='int32', mode='r') + # d = x[0] + # return x.view('float32').reshape(-1, d + 1)[:, 1:] - print(mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs")) + # print(mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs")) # SIFT_SRC_QUERY_DATA_DIR = '/poc/yuncong/ann_1000m' # file_name = SIFT_SRC_QUERY_DATA_DIR+'/'+'query.npy' # data = numpy.load(file_name) @@ -229,16 +263,21 @@ if __name__ == "__main__": # return x.view('float32').reshape(-1, d + 1)[:, 1:] # data = mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs") - # print(data[0], len(data[0]), len(data)) + # data = sklearn.preprocessing.normalize(data, axis=1, norm='l2') + # np.save("/test/milvus/deep1b/query.npy", data) - # total_size = 10000 - # # total_size = 1000000000 - # file_size = 1000 - # # file_size = 100000 - # file_num = total_size // file_size - # for i in range(file_num): - # fname = "/test/milvus/raw_data/deep1b/binary_96_%05d" % i - # print(fname, i*file_size, (i+1)*file_size) - # single_data = data[i*file_size : (i+1)*file_size] - # single_data = sklearn.preprocessing.normalize(single_data, axis=1, norm='l2') - # np.save(fname, single_data) + total_size = 100000000 + # total_size = 1000000000 + file_size = 100000 + # file_size = 100000 + dimension = 4096 + file_num = total_size // file_size + for i in range(file_num): + print(i) + # fname = "/test/milvus/raw_data/deep1b/binary_96_%05d" % i + fname = "/test/milvus/raw_data/random/binary_%dd_%05d" % (dimension, i) + # print(fname, i*file_size, (i+1)*file_size) + # single_data = data[i*file_size : (i+1)*file_size] + single_data = [[random.random() for _ in range(dimension)] for _ in range(file_size)] + single_data = sklearn.preprocessing.normalize(single_data, axis=1, norm='l2') + np.save(fname, single_data) diff --git a/tests/milvus_benchmark/conf/server_config.yaml b/tests/milvus_benchmark/conf/server_config.yaml index a5d2081b8a..592fe1e9aa 100644 --- a/tests/milvus_benchmark/conf/server_config.yaml +++ b/tests/milvus_benchmark/conf/server_config.yaml @@ -1,28 +1,34 @@ -cache_config: - cache_insert_data: false - cpu_cache_capacity: 16 - gpu_cache_capacity: 6 - cpu_cache_threshold: 0.85 -db_config: - backend_url: sqlite://:@:/ - build_index_gpu: 0 - insert_buffer_size: 4 - preload_table: null - primary_path: /opt/milvus - secondary_path: null -engine_config: - use_blas_threshold: 20 -metric_config: - collector: prometheus - enable_monitor: true - prometheus_config: - port: 8080 -resource_config: - resource_pool: - - cpu - - gpu0 server_config: - address: 0.0.0.0 - deploy_mode: single - port: 19530 - time_zone: UTC+8 + address: 0.0.0.0 + port: 19530 + deploy_mode: single + time_zone: UTC+8 + +db_config: + primary_path: /var/lib/milvus + secondary_path: + backend_url: sqlite://:@:/ + insert_buffer_size: 4 + preload_table: + +metric_config: + enable_monitor: true + collector: prometheus + prometheus_config: + port: 8080 + +cache_config: + cpu_cache_capacity: 16 + cache_insert_data: false + +engine_config: + use_blas_threshold: 1100 + gpu_search_threshold: 1000 +gpu_resource_config: + enable: true + cache_capacity: 4 + search_resources: + - gpu0 + build_index_resources: + - gpu0 + diff --git a/tests/milvus_benchmark/docker_runner.py b/tests/milvus_benchmark/docker_runner.py index 008c9866b4..7bedbf497c 100644 --- a/tests/milvus_benchmark/docker_runner.py +++ b/tests/milvus_benchmark/docker_runner.py @@ -28,6 +28,9 @@ class DockerRunner(Runner): container = None if op_type == "insert": + if not run_params: + logger.debug("No run params") + continue for index, param in enumerate(run_params): logger.info("Definition param: %s" % str(param)) table_name = param["table_name"] @@ -46,11 +49,13 @@ class DockerRunner(Runner): milvus.delete() time.sleep(10) milvus.create_table(table_name, dimension, index_file_size, metric_type) + # debug + # milvus.create_index("ivf_sq8", 16384) res = self.do_insert(milvus, table_name, data_type, dimension, table_size, param["ni_per"]) logger.info(res) # wait for file merge - time.sleep(6 * (table_size / 500000)) + time.sleep(table_size * dimension / 5000000) # Clear up utils.remove_container(container) @@ -80,9 +85,11 @@ class DockerRunner(Runner): for nlist in nlists: result = milvus.describe_index() logger.info(result) - milvus.create_index(index_type, nlist) + # milvus.drop_index() + # milvus.create_index(index_type, nlist) result = milvus.describe_index() logger.info(result) + logger.info(milvus.count()) # preload index milvus.preload_table() logger.info("Start warm up query") @@ -92,7 +99,7 @@ class DockerRunner(Runner): for nprobe in nprobes: logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count) - headers = ["Nprobe/Top-k"] + headers = ["Nq/Top-k"] headers.extend([str(top_k) for top_k in top_ks]) utils.print_table(headers, nqs, res) utils.remove_container(container) @@ -172,7 +179,18 @@ class DockerRunner(Runner): avg_acc = self.compute_accuracy(base_name, id_prefix) logger.info("Query: <%s> accuracy: %s" % (id_prefix, avg_acc)) else: - result_ids = self.do_query_ids(milvus, table_name, top_k, nq, nprobe) + result_ids, result_distances = self.do_query_ids(milvus, table_name, top_k, nq, nprobe) + debug_file_ids = "0.5.3_result_ids" + debug_file_distances = "0.5.3_result_distances" + with open(debug_file_ids, "w+") as fd: + total = 0 + for index, item in enumerate(result_ids): + true_item = true_ids_all[:nq, :top_k].tolist()[index] + tmp = set(item).intersection(set(true_item)) + total = total + len(tmp) + fd.write("query: N-%d, intersection: %d, total: %d\n" % (index, len(tmp), total)) + fd.write("%s\n" % str(item)) + fd.write("%s\n" % str(true_item)) acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids) logger.info("Query: <%s> accuracy: %s" % (id_prefix, acc_value)) # # print accuracy table @@ -195,6 +213,7 @@ class DockerRunner(Runner): for index, param in enumerate(run_params): logger.info("Definition param: %s" % str(param)) table_name = param["dataset"] + index_type = param["index_type"] volume_name = param["db_path_prefix"] (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) @@ -223,7 +242,9 @@ class DockerRunner(Runner): start_time = time.time() insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(10000)] + i = 0 while time.time() < start_time + during_time: + i = i + 1 processes = [] # do query # for i in range(query_process_num): @@ -243,7 +264,7 @@ class DockerRunner(Runner): nqs = random.sample([x for x in range(1, 1000)], 3) nprobe = random.choice([x for x in range(1, 500)]) res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count) - if int(time.time() - start_time) % 120 == 0: + if i % 10 == 0: status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))]) if not status.OK(): logger.error(status) @@ -251,9 +272,10 @@ class DockerRunner(Runner): # if not status.OK(): # logger.error(status) # index_type = random.choice(["flat", "ivf_flat", "ivf_sq8"]) + milvus_instance.create_index(index_type, 16384) result = milvus.describe_index() logger.info(result) - milvus_instance.create_index("ivf_sq8", 16384) + # milvus_instance.create_index("ivf_sq8", 16384) utils.remove_container(container) else: diff --git a/tests/milvus_benchmark/local_runner.py b/tests/milvus_benchmark/local_runner.py index 1067f500bc..22390e5470 100644 --- a/tests/milvus_benchmark/local_runner.py +++ b/tests/milvus_benchmark/local_runner.py @@ -29,7 +29,6 @@ class LocalRunner(Runner): if op_type == "insert": for index, param in enumerate(run_params): table_name = param["table_name"] - # random_1m_100_512 (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) milvus = MilvusClient(table_name, ip=self.ip, port=self.port) # Check has table or not @@ -47,25 +46,110 @@ class LocalRunner(Runner): (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) milvus = MilvusClient(table_name, ip=self.ip, port=self.port) + logger.info(milvus.describe()) + logger.info(milvus.describe_index()) # parse index info index_types = param["index.index_types"] nlists = param["index.nlists"] # parse top-k, nq, nprobe top_ks, nqs, nprobes = parser.search_params_parser(param) + # milvus.drop_index() for index_type in index_types: for nlist in nlists: - milvus.create_index(index_type, nlist) + # milvus.create_index(index_type, nlist) # preload index + logger.info("Start preloading table") milvus.preload_table() + logger.info("End preloading table") # Run query test + logger.info("Start warm up query") + res = self.do_query(milvus, table_name, [1], [1], 1, 1) + logger.info("End warm up query") for nprobe in nprobes: logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count) - headers = [param["dataset"]] + headers = ["nq/topk"] headers.extend([str(top_k) for top_k in top_ks]) utils.print_table(headers, nqs, res) + elif run_type == "accuracy": + for op_type, op_value in definition.items(): + if op_type != "query": + logger.warning("invalid operation: %s in accuracy test, only support query operation" % op_type) + break + run_count = op_value["run_count"] + run_params = op_value["params"] + for index, param in enumerate(run_params): + logger.info("Definition param: %s" % str(param)) + table_name = param["dataset"] + sift_acc = False + if "sift_acc" in param: + sift_acc = param["sift_acc"] + (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + + milvus = MilvusClient(table_name, ip=self.ip, port=self.port) + logger.debug(milvus.show_tables()) + # Check has table or not + if not milvus.exists_table(): + logger.warning("Table %s not existed, continue exec next params ..." % table_name) + continue + + # parse index info + index_types = param["index.index_types"] + nlists = param["index.nlists"] + # parse top-k, nq, nprobe + top_ks, nqs, nprobes = parser.search_params_parser(param) + + if sift_acc is True: + # preload groundtruth data + true_ids_all = self.get_groundtruth_ids(table_size) + + acc_dict = {} + for index_type in index_types: + for nlist in nlists: + result = milvus.describe_index() + logger.info(result) + # milvus.drop_index() + milvus.create_index(index_type, nlist) + # preload index + milvus.preload_table() + # Run query test + for nprobe in nprobes: + logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) + for top_k in top_ks: + for nq in nqs: + result_ids = [] + id_prefix = "%s_index_%s_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \ + (table_name, index_type, nlist, metric_type, nprobe, top_k, nq) + if sift_acc is False: + self.do_query_acc(milvus, table_name, top_k, nq, nprobe, id_prefix) + if index_type != "flat": + # Compute accuracy + base_name = "%s_index_flat_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \ + (table_name, nlist, metric_type, nprobe, top_k, nq) + avg_acc = self.compute_accuracy(base_name, id_prefix) + logger.info("Query: <%s> accuracy: %s" % (id_prefix, avg_acc)) + else: + result_ids, result_distances = self.do_query_ids(milvus, table_name, top_k, nq, nprobe) + debug_file_ids = "0.5.3_result_ids" + debug_file_distances = "0.5.3_result_distances" + with open(debug_file_ids, "w+") as fd: + total = 0 + for index, item in enumerate(result_ids): + true_item = true_ids_all[:nq, :top_k].tolist()[index] + tmp = set(item).intersection(set(true_item)) + total = total + len(tmp) + fd.write("query: N-%d, intersection: %d, total: %d\n" % (index, len(tmp), total)) + fd.write("%s\n" % str(item)) + fd.write("%s\n" % str(true_item)) + acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids) + logger.info("Query: <%s> accuracy: %s" % (id_prefix, acc_value)) + # # print accuracy table + # headers = [table_name] + # headers.extend([str(top_k) for top_k in top_ks]) + # utils.print_table(headers, nqs, res) + elif run_type == "stability": for op_type, op_value in definition.items(): if op_type != "query": @@ -73,11 +157,12 @@ class LocalRunner(Runner): break run_count = op_value["run_count"] run_params = op_value["params"] - nq = 10000 + nq = 100000 for index, param in enumerate(run_params): logger.info("Definition param: %s" % str(param)) table_name = param["dataset"] + index_type = param["index_type"] (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) # set default test time @@ -90,7 +175,10 @@ class LocalRunner(Runner): query_process_num = 10 else: query_process_num = int(param["query_process_num"]) - milvus = MilvusClient(table_name) + milvus = MilvusClient(table_name, ip=self.ip, port=self.port) + logger.debug(milvus.show_tables()) + logger.debug(milvus.describe_index()) + logger.debug(milvus.count()) # Check has table or not if not milvus.exists_table(): logger.warning("Table %s not existed, continue exec next params ..." % table_name) @@ -98,8 +186,9 @@ class LocalRunner(Runner): start_time = time.time() insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)] + i = 0 while time.time() < start_time + during_time: - processes = [] + # processes = [] # # do query # for i in range(query_process_num): # milvus_instance = MilvusClient(table_name) @@ -113,20 +202,14 @@ class LocalRunner(Runner): # time.sleep(0.1) # for p in processes: # p.join() - milvus_instance = MilvusClient(table_name) + i = i + 1 + milvus_instance = MilvusClient(table_name, ip=self.ip, port=self.port) top_ks = random.sample([x for x in range(1, 100)], 4) - nqs = random.sample([x for x in range(1, 1000)], 3) - nprobe = random.choice([x for x in range(1, 500)]) - res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count) + nqs = random.sample([x for x in range(1, 200)], 3) + nprobe = random.choice([x for x in range(1, 100)]) + res = self.do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count) # milvus_instance = MilvusClient(table_name) status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))]) if not status.OK(): logger.error(status.message) - if (time.time() - start_time) % 300 == 0: - status = milvus_instance.drop_index() - if not status.OK(): - logger.error(status.message) - index_type = random.choice(["flat", "ivf_flat", "ivf_sq8"]) - status = milvus_instance.create_index(index_type, 16384) - if not status.OK(): - logger.error(status.message) + # status = milvus_instance.create_index(index_type, 16384) diff --git a/tests/milvus_benchmark/requirements.txt b/tests/milvus_benchmark/requirements.txt index 328cff1eb4..ee6a8a11ff 100644 --- a/tests/milvus_benchmark/requirements.txt +++ b/tests/milvus_benchmark/requirements.txt @@ -1,6 +1,9 @@ numpy==1.16.3 -pymilvus>=0.1.18 -pyyaml==5.1 -docker==4.0.2 +pymilvus-test>=0.2.0 +scikit-learn==0.19.1 +h5py==2.7.1 +influxdb==5.2.2 +pyyaml>=5.1 tableprint==0.8.0 -ansicolors==1.1.8 \ No newline at end of file +ansicolors==1.1.8 +scipy==1.3.1 diff --git a/tests/milvus_benchmark/runner.py b/tests/milvus_benchmark/runner.py index d3ad345da9..76851fa69c 100644 --- a/tests/milvus_benchmark/runner.py +++ b/tests/milvus_benchmark/runner.py @@ -18,15 +18,18 @@ SIFT_VECTORS_PER_FILE = 100000 MAX_NQ = 10001 FILE_PREFIX = "binary_" -RANDOM_SRC_BINARY_DATA_DIR = '/tmp/random/binary_data' -SIFT_SRC_DATA_DIR = '/tmp/sift1b/query' -SIFT_SRC_BINARY_DATA_DIR = '/tmp/sift1b/binary_data' -SIFT_SRC_GROUNDTRUTH_DATA_DIR = '/tmp/sift1b/groundtruth' +# FOLDER_NAME = 'ann_1000m/source_data' +SRC_BINARY_DATA_DIR = '/poc/yuncong/yunfeng/random_data' +SRC_BINARY_DATA_DIR_high = '/test/milvus/raw_data/random' +SIFT_SRC_DATA_DIR = '/poc/yuncong/ann_1000m/' +SIFT_SRC_BINARY_DATA_DIR = SIFT_SRC_DATA_DIR + 'source_data' +SIFT_SRC_GROUNDTRUTH_DATA_DIR = SIFT_SRC_DATA_DIR + 'gnd' WARM_TOP_K = 1 WARM_NQ = 1 DEFAULT_DIM = 512 + GROUNDTRUTH_MAP = { "1000000": "idx_1M.ivecs", "2000000": "idx_2M.ivecs", @@ -45,7 +48,10 @@ def gen_file_name(idx, table_dimension, data_type): s = "%05d" % idx fname = FILE_PREFIX + str(table_dimension) + "d_" + s + ".npy" if data_type == "random": - fname = RANDOM_SRC_BINARY_DATA_DIR+'/'+fname + if table_dimension == 512: + fname = SRC_BINARY_DATA_DIR+'/'+fname + elif table_dimension >= 4096: + fname = SRC_BINARY_DATA_DIR_high+'/'+fname elif data_type == "sift": fname = SIFT_SRC_BINARY_DATA_DIR+'/'+fname return fname @@ -87,7 +93,12 @@ class Runner(object): qps = 0.0 ni_time = 0.0 if data_type == "random": - vectors_per_file = VECTORS_PER_FILE + if dimension == 512: + vectors_per_file = VECTORS_PER_FILE + elif dimension == 4096: + vectors_per_file = 100000 + elif dimension == 16384: + vectors_per_file = 10000 elif data_type == "sift": vectors_per_file = SIFT_VECTORS_PER_FILE if size % vectors_per_file or ni > vectors_per_file: @@ -101,15 +112,16 @@ class Runner(object): loops = vectors_per_file // ni for j in range(loops): vectors = data[j*ni:(j+1)*ni].tolist() - ni_start_time = time.time() - # start insert vectors - start_id = i * vectors_per_file + j * ni - end_id = start_id + len(vectors) - logger.info("Start id: %s, end id: %s" % (start_id, end_id)) - ids = [k for k in range(start_id, end_id)] - status, ids = milvus.insert(vectors, ids=ids) - ni_end_time = time.time() - total_time = total_time + ni_end_time - ni_start_time + if vectors: + ni_start_time = time.time() + # start insert vectors + start_id = i * vectors_per_file + j * ni + end_id = start_id + len(vectors) + logger.info("Start id: %s, end id: %s" % (start_id, end_id)) + ids = [k for k in range(start_id, end_id)] + status, ids = milvus.insert(vectors, ids=ids) + ni_end_time = time.time() + total_time = total_time + ni_end_time - ni_start_time qps = round(size / total_time, 2) ni_time = round(total_time / (loops * file_num), 2) @@ -125,10 +137,10 @@ class Runner(object): bi_res = [] for index, nq in enumerate(nqs): tmp_res = [] + vectors = base_query_vectors[0:nq] for top_k in top_ks: avg_query_time = 0.0 total_query_time = 0.0 - vectors = base_query_vectors[0:nq] logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors))) for i in range(run_count): logger.info("Start run query, run %d of %s" % (i+1, run_count)) @@ -153,12 +165,16 @@ class Runner(object): msg = "Query failed with message: %s" % status.message raise Exception(msg) result_ids = [] + result_distances = [] for result in query_res: tmp = [] + tmp_distance = [] for item in result: tmp.append(item.id) + tmp_distance.append(item.distance) result_ids.append(tmp) - return result_ids + result_distances.append(tmp_distance) + return result_ids, result_distances def do_query_acc(self, milvus, table_name, top_k, nq, nprobe, id_store_name): (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) @@ -216,4 +232,4 @@ class Runner(object): a = np.fromfile(fname, dtype='int32') d = a[0] true_ids = a.reshape(-1, d + 1)[:, 1:].copy() - return true_ids + return true_ids \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/accuracy_random50m.yaml b/tests/milvus_benchmark/suites/accuracy_random50m.yaml new file mode 100644 index 0000000000..d84c9e301c --- /dev/null +++ b/tests/milvus_benchmark/suites/accuracy_random50m.yaml @@ -0,0 +1,57 @@ + +accuracy: + # interface: search_vectors + query: + [ + { + "dataset": "random_50m_1024_512_ip", + "index.index_types": ["flat", "ivf_flat"], + "index.nlists": [16384], + "index.metric_types": ["ip"], + "nprobes": [8, 64, 128, 512, 2048], + "top_ks": [64], + "nqs": [1000], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 100, + "server.cpu_cache_capacity": 120, + "server.gpu_cache_capacity": 6, + "server.enable_gpu": True, + "server.resources": ["gpu0", "gpu1"], + "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_ivf", + "sift_acc": false + }, + { + "dataset": "random_50m_1024_512_ip", + "index.index_types": ["flat", "ivf_sq8"], + "index.nlists": [16384], + "index.metric_types": ["ip"], + "nprobes": [8, 64, 128, 512, 2048], + "top_ks": [64], + "nqs": [1000], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 100, + "server.cpu_cache_capacity": 120, + "server.gpu_cache_capacity": 6, + "server.enable_gpu": True, + "server.resources": ["gpu0", "gpu1"], + "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_sq8", + "sift_acc": false + }, + { + "dataset": "random_50m_1024_512_ip", + "index.index_types": ["flat", "ivf_sq8h"], + "index.nlists": [16384], + "index.metric_types": ["ip"], + "nprobes": [8, 64, 128, 512, 2048], + "top_ks": [64], + "nqs": [1000], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 100, + "server.cpu_cache_capacity": 120, + "server.gpu_cache_capacity": 6, + "server.enable_gpu": True, + "server.resources": ["gpu0", "gpu1"], + "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip", + "sift_acc": false + }, + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/accuracy_sift10m.yaml b/tests/milvus_benchmark/suites/accuracy_sift10m.yaml new file mode 100644 index 0000000000..35074868f1 --- /dev/null +++ b/tests/milvus_benchmark/suites/accuracy_sift10m.yaml @@ -0,0 +1,75 @@ +accuracy: + + # interface: search_vectors + query: + # dataset: table name you have already created + # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. + [ + # { + # "dataset": "sift_10m_1024_128_l2", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "index.metric_types": ["l2"], + # "nprobes": [1, 8, 16, 32, 64, 128, 256], + # "top_ks": [64], + # "nqs": [1000], + # "server.cpu_cache_capacity": 50, + # "server.use_gpu_threshold": 1001, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["cpu", "gpu0"], + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8h", + # "sift_acc": true + # }, + # { + # "dataset": "sift_10m_1024_128_l2", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "index.metric_types": ["l2"], + # "nprobes": [1, 8, 16, 32, 64, 128, 256], + # "top_ks": [64], + # "nqs": [1000], + # "server.cpu_cache_capacity": 50, + # "server.resources": ["cpu", "gpu0"], + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8", + # "sift_acc": true + # }, + { + "dataset": "sift_10m_1024_128_l2", + "index.index_types": ["ivf_pq"], + "index.nlists": [16384], + "index.metric_types": ["l2"], + "nprobes": [1, 8, 16, 32, 64, 128, 256], + "top_ks": [64], + "nqs": [1000], + "server.cpu_cache_capacity": 50, + "server.resources": ["cpu", "gpu0"], + "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8", + "sift_acc": true + }, + # { + # "dataset": "sift_10m_1024_128_l2", + # "index.index_types": ["ivf_flat"], + # "index.nlists": [16384], + # "index.metric_types": ["l2"], + # "nprobes": [1, 8, 16, 32, 64, 128, 256], + # "top_ks": [64], + # "nqs": [1000], + # "server.cpu_cache_capacity": 100, + # "server.resources": ["cpu", "gpu0"], + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_ivf", + # "sift_acc": true + # }, + # { + # "dataset": "sift_10m_1024_128_l2", + # "index.index_types": ["nsg"], + # "index.nlists": [16384], + # "index.metric_types": ["l2"], + # "nprobes": [1], + # "top_ks": [64], + # "nqs": [1000], + # # "server.cpu_cache_capacity": 100, + # # "server.resources": ["cpu", "gpu0"], + # "db_path_prefix": "/test/milvus/db_data_gpu/sift_10m_1024_128_l2_nsg", + # "sift_acc": true + # }, + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/accuracy_sift1b.yaml b/tests/milvus_benchmark/suites/accuracy_sift1b.yaml new file mode 100644 index 0000000000..7ad6e85da3 --- /dev/null +++ b/tests/milvus_benchmark/suites/accuracy_sift1b.yaml @@ -0,0 +1,73 @@ + +accuracy: + # interface: search_vectors + query: + [ + # { + # "dataset": "sift_1b_2048_128_l2", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "index.metric_types": ["l2"], + # # "nprobes": [1, 8, 32, 64, 128], + # "nprobes": [1, 8, 32, 64, 128, 256], + # "top_ks": [64], + # "nqs": [1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 1001, + # "server.cpu_cache_capacity": 150, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data_gpu/sift_1b_2048_128_l2", + # "sift_acc": true + # }, + { + "dataset": "sift_1b_2048_128_l2", + "index.index_types": ["ivf_pq"], + "index.nlists": [16384], + "index.metric_types": ["l2"], + "nprobes": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], + "top_ks": [64], + "nqs": [1000], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 10001, + "server.cpu_cache_capacity": 150, + "server.gpu_cache_capacity": 6, + "server.resources": ["gpu0", "gpu1"], + "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_pq", + "sift_acc": true + }, + + # { + # "dataset": "sift_1b_2048_128_l2", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "index.metric_types": ["l2"], + # "nprobes": [1, 8, 32, 64, 128], + # "top_ks": [64], + # "nqs": [1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 150, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2", + # "sift_acc": true + # }, + # { + # "dataset": "sift_1b_2048_128_l2", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "index.metric_types": ["l2"], + # "nprobes": [1, 8, 32, 64, 128], + # "top_ks": [64], + # "nqs": [1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 150, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["cpu", "gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2", + # "sift_acc": true + # }, + + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/accuracy_sift500m.yaml b/tests/milvus_benchmark/suites/accuracy_sift500m.yaml new file mode 100644 index 0000000000..5977fbbc89 --- /dev/null +++ b/tests/milvus_benchmark/suites/accuracy_sift500m.yaml @@ -0,0 +1,21 @@ +accuracy: + + # interface: search_vectors + query: + # dataset: table name you have already created + # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. + [ + { + "dataset": "sift_500m_1024_128_l2", + "index.index_types": ["ivf_sq8h"], + "index.nlists": [16384], + "index.metric_types": ["l2"], + "nprobes": [1, 8, 16, 32, 64, 128, 256], + "top_ks": [64], + "nqs": [1000], + "server.cpu_cache_capacity": 200, + "server.resources": ["cpu", "gpu0"], + "db_path_prefix": "/test/milvus/db_data/sift_500m_1024_128_l2", + "sift_acc": true + }, + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/accuracy_sift50m.yaml b/tests/milvus_benchmark/suites/accuracy_sift50m.yaml new file mode 100644 index 0000000000..1f4dadbb00 --- /dev/null +++ b/tests/milvus_benchmark/suites/accuracy_sift50m.yaml @@ -0,0 +1,76 @@ +accuracy: + + # interface: search_vectors + query: + # dataset: table name you have already created + # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. + [ + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "index.metric_types": ["l2"], + # "nprobes": [1, 8, 16, 32, 64, 128, 256], + # "top_ks": [64], + # "nqs": [1000], + # "server.cpu_cache_capacity": 50, + # "server.use_gpu_threshold": 1001, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["cpu", "gpu0"], + # "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8h", + # "sift_acc": true + # }, + { + "dataset": "sift_50m_1024_128_l2", + "index.index_types": ["ivf_sq8"], + "index.nlists": [16384], + "index.metric_types": ["l2"], + "nprobes": [1, 8, 16, 32, 64, 128, 256], + "top_ks": [64], + "nqs": [1000], + "server.cpu_cache_capacity": 50, + "server.resources": ["cpu", "gpu0"], + "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8", + "sift_acc": true + }, + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["ivf_flat"], + # "index.nlists": [16384], + # "index.metric_types": ["l2"], + # "nprobes": [1, 8, 16, 32, 64, 128, 256], + # "top_ks": [64], + # "nqs": [1000], + # "server.cpu_cache_capacity": 100, + # "server.resources": ["cpu", "gpu0"], + # "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_ivf", + # "sift_acc": true + # }, + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["ivf_flat"], + # "index.nlists": [16384], + # "index.metric_types": ["l2"], + # "nprobes": [1, 8, 16, 32, 64, 128, 256], + # "top_ks": [64], + # "nqs": [1000], + # "server.cpu_cache_capacity": 100, + # "server.resources": ["cpu", "gpu0"], + # "db_path_prefix": "/test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf", + # "sift_acc": true + # }, + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["nsg"], + # "index.nlists": [16384], + # "index.metric_types": ["l2"], + # "nprobes": [1], + # "top_ks": [64], + # "nqs": [1000], + # "server.cpu_cache_capacity": 100, + # "server.resources": ["gpu0"], + # "db_path_prefix": "/test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg", + # "sift_acc": true + # }, + + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert.yaml b/tests/milvus_benchmark/suites/insert.yaml new file mode 100644 index 0000000000..767ad481c5 --- /dev/null +++ b/tests/milvus_benchmark/suites/insert.yaml @@ -0,0 +1,32 @@ +performance: + + # interface: add_vectors + insert: + # index_type: flat/ivf_flat/ivf_sq8/mix_nsg + [ + # debug + # data_type / data_size / index_file_size / dimension + # data_type: random / ann_sift + # data_size: 10m / 1b + # { + # "table_name": "random_1m_1024_4096_ip", + # "ni_per": 10000, + # "server.cpu_cache_capacity": 16, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_1m_1024_4096_ip" + # }, + # { + # "table_name": "random_1m_1024_16384_ip", + # "ni_per": 10000, + # "server.cpu_cache_capacity": 16, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_1m_1024_16384_ip" + # }, + { + "table_name": "random_50m_1024_512_ip", + "ni_per": 100000, + # "server.cpu_cache_capacity": 16, + # "server.resources": ["gpu0", "gpu1"], + "db_path_prefix": "/test/milvus/db_data_cpu/random_50m_1024_512_ip_ivf" + }, + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/old_performance_sift50m.yaml b/tests/milvus_benchmark/suites/old_performance_sift50m.yaml new file mode 100644 index 0000000000..8e4d35dcac --- /dev/null +++ b/tests/milvus_benchmark/suites/old_performance_sift50m.yaml @@ -0,0 +1,80 @@ +performance: + + # interface: search_vectors + query: + # dataset: table name you have already created + # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. + [ + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["mix_nsg"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_nsg" + # }, + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["ivf_flat"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_ivf" + # }, + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.gpu_cache_capacity": 3, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8" + # }, + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8" + # }, + { + "dataset": "sift_50m_1024_128_l2", + "index.index_types": ["ivf_sq8"], + "index.nlists": [16384], + "nprobes": [8, 32], + "top_ks": [1, 16, 64, 128, 256, 512, 1000], + "nqs": [1, 10, 100, 200, 500, 1000], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 100, + "server.cpu_cache_capacity": 50, + "server.gpu_cache_capacity": 6, + "server.resources": ["gpu0", "gpu1"], + "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8" + }, + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_random1m.yaml b/tests/milvus_benchmark/suites/performance_random1m.yaml new file mode 100644 index 0000000000..606f721567 --- /dev/null +++ b/tests/milvus_benchmark/suites/performance_random1m.yaml @@ -0,0 +1,35 @@ +performance: + + # interface: search_vectors + query: + # dataset: table name you have already created + # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. + [ + # { + # "dataset": "random_1m_1024_4096_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 1100, + # "server.cpu_cache_capacity": 50, + # "server.resources": ["cpu", "gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_1m_1024_4096_ip_sq8h" + # }, + { + "dataset": "random_2m_1024_512_ip", + "index.index_types": ["mix_nsg"], + "index.nlists": [16384], + "nprobes": [8], + "top_ks": [1], + "nqs": [1], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 1100, + "server.cpu_cache_capacity": 50, + "server.resources": ["gpu0", "gpu1"], + "server.enable_gpu": True, + "db_path_prefix": "/test/milvus/db_data/random_1m_1024_4096_ip_sq8" + }, + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_random50m.yaml b/tests/milvus_benchmark/suites/performance_random50m.yaml new file mode 100644 index 0000000000..20cb78c099 --- /dev/null +++ b/tests/milvus_benchmark/suites/performance_random50m.yaml @@ -0,0 +1,177 @@ +performance: + + # interface: search_vectors + query: + # dataset: table name you have already created + # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. + [ + # { + # "dataset": "random_50m_2048_512_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.resources": ["cpu", "gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h" + # }, + # { + # "dataset": "random_50m_2048_512_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 1000, + # "server.cpu_cache_capacity": 50, + # "server.resources": ["cpu", "gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h" + # }, + # { + # "dataset": "random_50m_2048_512_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 10000, + # "server.cpu_cache_capacity": 50, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h" + # }, + # { + # "dataset": "random_50m_2048_512_ip", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8" + # }, + { + "dataset": "random_50m_2048_512_ip", + "index.index_types": ["ivf_sq8"], + "index.nlists": [16384], + "nprobes": [8], + "top_ks": [64], + "nqs": [1, 10, 100], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 100, + "server.cpu_cache_capacity": 50, + "server.enable_gpu": True, + "server.resources": ["gpu0", "gpu1"], + "db_path_prefix": "/test/milvus/db_data_cpu/random_50m_2048_512_ip_sq8" + }, + + # { + # "dataset": "random_50m_1024_512_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 110, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" + # }, + # { + # "dataset": "random_50m_1024_512_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 1000, + # "server.cpu_cache_capacity": 110, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" + # }, + # { + # "dataset": "random_50m_1024_512_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 10000, + # "server.cpu_cache_capacity": 110, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" + # }, + # { + # "dataset": "random_50m_1024_512_ip", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.cpu_cache_capacity": 110, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8" + # }, + # { + # "dataset": "random_50m_1024_512_ip", + # "index.index_types": ["ivf_pq"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 110, + # "server.gpu_cache_capacity": 6, + # "server.index_build_device": "gpu0", + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_pq" + # }, + # { + # "dataset": "random_50m_1024_512_ip", + # "index.index_types": ["ivf_flat"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 110, + # "server.gpu_cache_capacity": 6, + # "server.index_build_device": "gpu0", + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_ivf" + # }, + # { + # "dataset": "random_50m_1024_512_ip", + # "index.index_types": ["nsg"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 110, + # "server.gpu_cache_capacity": 6, + # "server.index_build_device": "gpu0", + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_nsg" + # } + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_sift10m.yaml b/tests/milvus_benchmark/suites/performance_sift10m.yaml new file mode 100644 index 0000000000..e98d54bc3a --- /dev/null +++ b/tests/milvus_benchmark/suites/performance_sift10m.yaml @@ -0,0 +1,83 @@ +performance: + + # interface: search_vectors + query: + # dataset: table name you have already created + # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. + [ + # { + # "dataset": "sift_10m_1024_128_l2", + # "index.index_types": ["ivf_flat"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0"], + # "server.enable_gpu": False, + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_ivf" + # }, + # { + # "dataset": "sift_10m_1024_128_l2", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0"], + # "server.enable_gpu": False, + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_sq8" + # }, + # { + # "dataset": "sift_10m_1024_128_l2", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0"], + # "server.enable_gpu": False, + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_sq8h" + # }, + { + "dataset": "sift_10m_1024_128_l2", + "index.index_types": ["ivf_pq"], + "index.nlists": [16384], + "nprobes": [8, 32], + "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + "nqs": [1, 10, 100, 500, 800, 1000], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 100, + "server.cpu_cache_capacity": 50, + "server.gpu_cache_capacity": 6, + "server.resources": ["gpu0"], + "server.enable_gpu": False, + "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_pq" + }, + # { + # "dataset": "sift_10m_1024_128_l2", + # "index.index_types": ["nsg"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0"], + # "server.enable_gpu": False, + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_nsg" + # }, + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_sift1b.yaml b/tests/milvus_benchmark/suites/performance_sift1b.yaml new file mode 100644 index 0000000000..505bf726af --- /dev/null +++ b/tests/milvus_benchmark/suites/performance_sift1b.yaml @@ -0,0 +1,123 @@ +performance: + + # interface: search_vectors + query: + # dataset: table name you have already created + # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. + [ + # { + # "dataset": "sift_1b_2048_128_l2", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 10000, + # "server.cpu_cache_capacity": 150, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h" + # }, + # { + # "dataset": "sift_1b_2048_128_l2", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 150, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h" + # }, + # { + # "dataset": "sift_1b_2048_128_l2", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 150, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["cpu", "gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h" + # }, + # { + # "dataset": "sift_1b_2048_128_l2", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 1000, + # "server.cpu_cache_capacity": 150, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h" + # }, + # { + # "dataset": "sift_1b_2048_128_l2", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 10000, + # "server.cpu_cache_capacity": 150, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": False, + # "db_path_prefix": "/test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h" + # }, + { + "dataset": "sift_1b_2048_128_l2", + "index.index_types": ["ivf_pq"], + "index.nlists": [16384], + "nprobes": [8, 32], + "top_ks": [1, 16, 64, 128, 256, 512, 1000], + "nqs": [1, 10, 100, 200, 500, 1000], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 10000, + "server.cpu_cache_capacity": 150, + "server.gpu_cache_capacity": 6, + "server.resources": ["gpu0", "gpu1"], + "server.enable_gpu": False, + "db_path_prefix": "/test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq" + }, + # { + # "dataset": "sift_1b_2048_128_l2", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 150, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": False, + # "db_path_prefix": "/test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8" + # }, + # { + # "dataset": "sift_1b_2048_128_l2", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 150, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["cpu", "gpu0"], + # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2" + # }, + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_sift1m.yaml b/tests/milvus_benchmark/suites/performance_sift1m.yaml new file mode 100644 index 0000000000..5c434b7d3f --- /dev/null +++ b/tests/milvus_benchmark/suites/performance_sift1m.yaml @@ -0,0 +1,23 @@ +performance: + + # interface: search_vectors + query: + # dataset: table name you have already created + # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. + [ + { + "dataset": "sift_1m_1024_128_l2", + "index.index_types": ["mix_nsg"], + "index.nlists": [16384], + "nprobes": [8, 32], + "top_ks": [1, 16, 64, 128, 256, 512, 1000], + "nqs": [1, 10, 100, 200, 500, 1000], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 100, + "server.cpu_cache_capacity": 50, + "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + "server.enable_gpu": True, + "db_path_prefix": "/test/milvus/db_data/sift_1m_1024_128_l2_nsg" + }, + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_sift500m.yaml b/tests/milvus_benchmark/suites/performance_sift500m.yaml new file mode 100644 index 0000000000..99ab7a3204 --- /dev/null +++ b/tests/milvus_benchmark/suites/performance_sift500m.yaml @@ -0,0 +1,50 @@ +performance: + + # interface: search_vectors + query: + # dataset: table name you have already created + # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. + [ + # { + # "dataset": "sift_500m_1024_128_l2", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 150, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/sift_500m_1024_128_l2_sq8" + # }, + # { + # "dataset": "sift_500m_1024_128_l2", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 150, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/sift_500m_1024_128_l2_sq8" + # }, + { + "dataset": "sift_500m_1024_128_l2", + "index.index_types": ["ivf_sq8h"], + "index.nlists": [16384], + "nprobes": [8, 32], + "top_ks": [1, 100, 200, 1000], + "nqs": [1, 10, 100, 500, 1000], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 100, + "server.cpu_cache_capacity": 150, + "server.gpu_cache_capacity": 6, + "server.resources": ["gpu0", "gpu1"], + "db_path_prefix": "/test/milvus/db_data/sift_500m_1024_128_l2" + }, + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_sift50m.yaml b/tests/milvus_benchmark/suites/performance_sift50m.yaml new file mode 100644 index 0000000000..85fece5b56 --- /dev/null +++ b/tests/milvus_benchmark/suites/performance_sift50m.yaml @@ -0,0 +1,82 @@ +performance: + + # interface: search_vectors + query: + # dataset: table name you have already created + # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. + [ + { + "dataset": "sift_50m_1024_128_l2", + "index.index_types": ["nsg"], + "index.nlists": [16384], + "nprobes": [8], + "top_ks": [64], + "nqs": [1, 10, 100], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 100, + "server.cpu_cache_capacity": 50, + "server.gpu_cache_capacity": 6, + "server.resources": ["gpu0", "gpu1"], + "server.enable_gpu": True, + "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_nsg" + }, + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["ivf_flat"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_ivf" + # }, + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.gpu_cache_capacity": 3, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h" + # }, + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8" + # }, + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["ivf_pq"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 200, 500, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.gpu_cache_capacity": 6, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_pq" + # }, + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/suites.yaml b/tests/milvus_benchmark/suites/suites.yaml new file mode 100644 index 0000000000..475020aa8e --- /dev/null +++ b/tests/milvus_benchmark/suites/suites.yaml @@ -0,0 +1,161 @@ +performance: + + # interface: search_vectors + query: + # dataset: table name you have already created + # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. + [ + # { + # "dataset": "random_50m_2048_512_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.resources": ["cpu", "gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h" + # }, + # { + # "dataset": "random_50m_2048_512_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 1000, + # "server.cpu_cache_capacity": 50, + # "server.resources": ["cpu", "gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h" + # }, + # { + # "dataset": "random_50m_2048_512_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 10000, + # "server.cpu_cache_capacity": 50, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h" + # }, + # { + # "dataset": "random_50m_2048_512_ip", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8" + # }, + # { + # "dataset": "random_50m_2048_512_ip", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 50, + # "server.enable_gpu": True, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data_cpu/random_50m_2048_512_ip_sq8" + # }, + + # { + # "dataset": "random_50m_1024_512_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 110, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" + # }, + # { + # "dataset": "random_50m_1024_512_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 1000, + # "server.cpu_cache_capacity": 110, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" + # }, + # { + # "dataset": "random_50m_1024_512_ip", + # "index.index_types": ["ivf_sq8h"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 10000, + # "server.cpu_cache_capacity": 110, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" + # }, + # { + # "dataset": "random_50m_1024_512_ip", + # "index.index_types": ["ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [8, 32], + # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + # "nqs": [1, 10, 100, 500, 800, 1000], + # "server.use_blas_threshold": 1100, + # "server.cpu_cache_capacity": 110, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8" + # }, + { + "dataset": "random_50m_1024_512_ip", + "index.index_types": ["ivf_pq"], + "index.nlists": [16384], + "nprobes": [8, 32], + "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], + "nqs": [1, 10, 100, 500, 800, 1000], + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 100, + "server.cpu_cache_capacity": 110, + "server.gpu_cache_capacity": 6, + "server.index_build_device": "gpu0", + "server.resources": ["gpu0", "gpu1"], + "server.enable_gpu": True, + "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_pq" + }, + # { + # "dataset": "random_50m_1024_512_ip", + # "index.index_types": ["ivf_flat"], + # "index.nlists": [16384], + # "nprobes": [8], + # "top_ks": [8], + # "nqs": [500], + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.cpu_cache_capacity": 110, + # "server.gpu_cache_capacity": 6, + # "server.index_build_device": "gpu0", + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": True, + # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_ivf" + # } + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/suites_accuracy.yaml b/tests/milvus_benchmark/suites/suites_accuracy.yaml new file mode 100644 index 0000000000..3dd91d2c02 --- /dev/null +++ b/tests/milvus_benchmark/suites/suites_accuracy.yaml @@ -0,0 +1,51 @@ + +accuracy: + # interface: search_vectors + query: + [ + # { + # "dataset": "random_20m_1024_512_ip", + # # index info + # "index.index_types": ["flat", "ivf_sq8"], + # "index.nlists": [16384], + # "index.metric_types": ["ip"], + # "nprobes": [1, 16, 64], + # "top_ks": [64], + # "nqs": [100], + # "server.cpu_cache_capacity": 100, + # "server.resources": ["gpu0"], + # "db_path_prefix": "/test/milvus/db_data/random_20m_1024_512_ip", + # }, + # { + # "dataset": "sift_1m_1024_128_l2", + # "index.index_types": ["flat", "ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [1, 32, 128, 256, 512], + # "nqs": 10, + # "top_ks": 10, + # "server.use_blas_threshold": 1100, + # "server.cpu_cache_capacity": 16, + # }, + # { + # "dataset": "sift_10m_1024_128_l2", + # "index.index_types": ["flat", "ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [1, 32, 128, 256, 512], + # "nqs": 10, + # "top_ks": 10, + # "server.use_blas_threshold": 1100, + # "server.cpu_cache_capacity": 32, + # }, + # { + # "dataset": "sift_50m_1024_128_l2", + # "index.index_types": ["flat", "ivf_sq8"], + # "index.nlists": [16384], + # "nprobes": [1, 32, 128, 256, 512], + # "nqs": 10, + # "top_ks": 10, + # "server.use_blas_threshold": 1100, + # "server.cpu_cache_capacity": 64, + # } + + + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/suites_performance.yaml b/tests/milvus_benchmark/suites/suites_performance.yaml new file mode 100644 index 0000000000..50100f4646 --- /dev/null +++ b/tests/milvus_benchmark/suites/suites_performance.yaml @@ -0,0 +1,57 @@ +performance: + + # interface: add_vectors + insert: + # index_type: flat/ivf_flat/ivf_sq8/mix_nsg + [ + # debug + # data_type / data_size / index_file_size / dimension + # data_type: random / ann_sift + # data_size: 10m / 1b + # { + # "table_name": "random_50m_1024_512_ip", + # "ni_per": 100000, + # "server.cpu_cache_capacity": 16, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data" + # }, + # { + # "table_name": "random_2m_1024_512_ip", + # "ni_per": 100000, + # "processes": 5, # multiprocessing + # "server.cpu_cache_capacity": 16, + # "server.resources": ["gpu0", "gpu1"], + # "db_path_prefix": "/test/milvus/db_data/random_5m_1024_512_ip" + # }, + # { + # "table_name": "sift_1m_50_128_l2", + # "ni_per": 100000, + # "processes": 5, # multiprocessing + # # "server.cpu_cache_capacity": 16, + # "db_path_prefix": "/test/milvus/db_data" + # }, + # { + # "table_name": "sift_1m_256_128_l2", + # "ni_per": 100000, + # "processes": 5, # multiprocessing + # # "server.cpu_cache_capacity": 16, + # "db_path_prefix": "/test/milvus/db_data" + # } + # { + # "table_name": "sift_50m_1024_128_l2", + # "ni_per": 100000, + # "processes": 5, # multiprocessing + # # "server.cpu_cache_capacity": 16, + # }, + # { + # "table_name": "sift_100m_1024_128_l2", + # "ni_per": 100000, + # "processes": 5, # multiprocessing + # }, + # { + # "table_name": "sift_1b_2048_128_l2", + # "ni_per": 100000, + # "processes": 5, # multiprocessing + # "server.cpu_cache_capacity": 16, + # } + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/suites_stability.yaml b/tests/milvus_benchmark/suites/suites_stability.yaml new file mode 100644 index 0000000000..427f48d924 --- /dev/null +++ b/tests/milvus_benchmark/suites/suites_stability.yaml @@ -0,0 +1,31 @@ + +stability: + # interface: search_vectors / add_vectors mix operation + query: + [ + # { + # "dataset": "random_50m_1024_512_ip", + # "index_type": "ivf_sq8", + # "query_process_num": 10, + # "during_time": 960, + # "server.cpu_cache_capacity": 100, + # "server.use_blas_threshold": 1100, + # "server.use_gpu_threshold": 100, + # "server.resources": ["gpu0", "gpu1"], + # "server.enable_gpu": false, + # "db_path_prefix": "/test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8_stability" + # }, + + { + "dataset": "random_50m_1024_512_ip", + "index_type": "ivf_sq8", + "query_process_num": 10, + "during_time": 960, + "server.cpu_cache_capacity": 100, + "server.use_blas_threshold": 1100, + "server.use_gpu_threshold": 100, + "server.resources": ["gpu0", "gpu1"], + "server.enable_gpu": false, + "db_path_prefix": "/test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8_stability" + }, + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/utils.py b/tests/milvus_benchmark/utils.py index f6522578ad..ddcd215237 100644 --- a/tests/milvus_benchmark/utils.py +++ b/tests/milvus_benchmark/utils.py @@ -45,12 +45,18 @@ def modify_config(k, v, type=None, file_path="conf/server_config.yaml", db_slave if config_dict: if k.find("use_blas_threshold") != -1: config_dict['engine_config']['use_blas_threshold'] = int(v) + elif k.find("use_gpu_threshold") != -1: + config_dict['engine_config']['gpu_search_threshold'] = int(v) elif k.find("cpu_cache_capacity") != -1: config_dict['cache_config']['cpu_cache_capacity'] = int(v) + elif k.find("enable_gpu") != -1: + config_dict['gpu_resource_config']['enable'] = v elif k.find("gpu_cache_capacity") != -1: - config_dict['cache_config']['gpu_cache_capacity'] = int(v) - elif k.find("resource_pool") != -1: - config_dict['resource_config']['resource_pool'] = v + config_dict['gpu_resource_config']['cache_capacity'] = int(v) + elif k.find("index_build_device") != -1: + config_dict['gpu_resource_config']['build_index_resources'] = v + elif k.find("search_resources") != -1: + config_dict['resource_config']['resources'] = v if db_slave: config_dict['db_config']['db_slave_path'] = MULTI_DB_SLAVE_PATH @@ -123,7 +129,7 @@ def run_server(image, mem_limit=None, timeout=30, test_type="local", volume_name volumes=volumes, runtime="nvidia", ports={'19530/tcp': 19530, '8080/tcp': 8080}, - environment=["OMP_NUM_THREADS=48"], + # environment=["OMP_NUM_THREADS=48"], # cpuset_cpus=cpu_limit, # mem_limit=mem_limit, # environment=[""],