From 8c563f1de054c03faafd990f67ca11cf53b6ee8e Mon Sep 17 00:00:00 2001 From: ThreadDao Date: Mon, 16 Nov 2020 15:28:26 +0800 Subject: [PATCH] [skip ci] update benchamrk version to 0.11.1 (#4236) Signed-off-by: ThreadDao --- tests/milvus_benchmark/.gitignore | 4 +- .../ci/jenkinsfile/cleanup.groovy | 8 +- .../ci/jenkinsfile/cleanupShards.groovy | 13 + .../ci/jenkinsfile/deploy_shards_test.groovy | 21 + .../ci/jenkinsfile/deploy_test.groovy | 36 +- .../ci/jenkinsfile/publishDailyImages.groovy | 2 +- tests/milvus_benchmark/ci/main_jenkinsfile | 39 +- tests/milvus_benchmark/ci/publish_jenkinsfile | 7 +- tests/milvus_benchmark/client.py | 563 +++++------ tests/milvus_benchmark/conf/log_config.conf | 28 - .../milvus_benchmark/conf/server_config.yaml | 34 - .../conf/server_config.yaml.cpu | 31 - .../conf/server_config.yaml.multi | 33 - .../conf/server_config.yaml.single | 32 - tests/milvus_benchmark/docker_utils.py | 125 +++ tests/milvus_benchmark/executors/__init__.py | 3 + tests/milvus_benchmark/executors/shell.py | 4 + tests/milvus_benchmark/handlers/__init__.py | 0 tests/milvus_benchmark/helm_utils.py | 326 +++++++ tests/milvus_benchmark/k8s_runner.py | 889 ++++++++++++++---- tests/milvus_benchmark/local_runner.py | 637 ++++++++++--- tests/milvus_benchmark/locust_file.py | 30 + tests/milvus_benchmark/locust_flush_task.py | 34 + .../locust_get_entity_task.py | 37 + tests/milvus_benchmark/locust_insert_task.py | 35 + tests/milvus_benchmark/locust_search_task.py | 38 + tests/milvus_benchmark/locust_task.py | 35 + tests/milvus_benchmark/locust_tasks.py | 41 + tests/milvus_benchmark/locust_user.py | 66 ++ tests/milvus_benchmark/main.py | 54 +- tests/milvus_benchmark/mix_task.py | 45 + tests/milvus_benchmark/parser.py | 6 +- tests/milvus_benchmark/report.py | 10 - tests/milvus_benchmark/requirements.txt | 5 +- tests/milvus_benchmark/results/__init__.py | 11 + tests/milvus_benchmark/results/reporter.py | 0 tests/milvus_benchmark/runner.py | 189 ++-- tests/milvus_benchmark/runners/__init__.py | 11 + .../milvus_benchmark/runners/locust_runner.py | 75 ++ .../milvus_benchmark/scheduler/010_data.json | 65 ++ .../milvus_benchmark/scheduler/011_data.json | 53 ++ .../scheduler/011_data_acc_debug.json | 11 + .../{ann_acc.json => 011_data_gpu_build.json} | 6 +- .../scheduler/011_data_insert.json | 11 + .../scheduler/011_data_search_debug.json | 11 + .../scheduler/011_delete.json | 15 + .../milvus_benchmark/scheduler/080_data.json | 34 +- tests/milvus_benchmark/scheduler/apollo.json | 6 +- tests/milvus_benchmark/scheduler/athena.json | 12 +- tests/milvus_benchmark/scheduler/build.json | 4 +- .../scheduler/{crud_flush.json => cqx.json} | 2 +- .../scheduler/crud_search.json | 11 - tests/milvus_benchmark/scheduler/debug.json | 4 +- tests/milvus_benchmark/scheduler/eros.json | 6 +- .../{crud_build.json => filter.json} | 3 +- tests/milvus_benchmark/scheduler/insert.json | 5 +- tests/milvus_benchmark/scheduler/jaccard.json | 4 +- .../scheduler/{crud_add.json => locust.json} | 2 +- .../{hnsw.json => locust_mix_debug.json} | 3 +- .../{crud_stability.json => loop.json} | 3 +- .../milvus_benchmark/scheduler/poseidon.json | 18 +- tests/milvus_benchmark/scheduler/search.json | 4 +- .../scheduler/search_performance.json | 11 - tests/milvus_benchmark/scheduler/shards.json | 18 + .../scheduler/shards_ann.json | 10 + .../scheduler/shards_debug.json | 15 + .../scheduler/shards_stability.json | 10 + tests/milvus_benchmark/scheduler/yx.json | 11 + .../scripts/default_config.json | 86 -- tests/milvus_benchmark/scripts/scheduler.py | 28 - tests/milvus_benchmark/search_task.py | 49 + .../suites/011_add_flush_performance.yaml | 20 + .../suites/011_build_debug.yaml | 92 ++ .../suites/011_compact_performance.yaml | 15 + .../suites/011_cpu_accuracy.yaml | 55 ++ .../suites/011_cpu_accuracy_ann.yaml | 271 ++++++ .../suites/011_cpu_accuracy_ann_debug.yaml | 73 ++ .../suites/011_cpu_accuracy_rhnsw.yaml | 36 + .../suites/011_cpu_build.yaml | 40 + .../suites/011_cpu_build_binary.yaml | 11 + .../suites/011_cpu_build_hnsw.yaml | 12 + .../suites/011_cpu_build_rhnsw.yaml | 23 + .../suites/011_cpu_search.yaml | 255 +++++ .../suites/011_cpu_search_binary.yaml | 49 + .../suites/011_cpu_search_sift10m.yaml | 122 +++ .../suites/011_cpu_search_sift10m_100k.yaml | 97 ++ .../suites/011_cpu_search_sift10m_filter.yaml | 97 ++ .../suites/011_cpu_search_sift10m_hnsw.yaml | 40 + .../suites/011_cpu_search_sift10m_ivf.yaml | 32 + .../suites/011_cpu_search_sift1b.yaml | 26 + .../suites/011_cpu_search_sift50m.yaml | 98 ++ .../suites/011_delete_performance.yaml | 17 + tests/milvus_benchmark/suites/011_get_id.yaml | 17 + .../suites/011_gpu_accuracy.yaml | 61 ++ .../suites/011_gpu_accuracy_ann.yaml | 172 ++++ .../suites/011_gpu_accuracy_ann_debug.yaml | 25 + .../suites/011_gpu_accuracy_debug.yaml | 23 + .../suites/011_gpu_build.yaml | 21 + .../suites/011_gpu_build_debug.yaml | 151 +++ .../suites/011_gpu_build_sift10m.yaml | 148 +++ .../suites/011_gpu_build_sift1b.yaml | 42 + .../suites/011_gpu_build_sift50m.yaml | 75 ++ .../suites/011_gpu_search.yaml | 225 +++++ .../suites/011_gpu_search_debug.yaml | 73 ++ .../suites/011_gpu_search_sift10m.yaml | 145 +++ .../suites/011_gpu_search_sift10m_100k.yaml | 121 +++ .../suites/011_gpu_search_sift10m_filter.yaml | 122 +++ .../suites/011_gpu_search_sift10m_ivf.yaml | 50 + .../suites/011_gpu_search_sift50m.yaml | 121 +++ .../suites/011_gpu_sift50m_ivf.yaml | 26 + .../suites/011_gpu_stability.yaml | 39 + .../suites/011_insert_data.yaml | 55 ++ .../suites/011_insert_debug.yaml | 89 ++ .../suites/011_insert_performance.yaml | 95 ++ .../suites/011_locust_search.yaml | 43 + .../suites/011_search_dsl.yaml | 76 ++ .../suites/011_search_stability.yaml | 20 + .../suites/011_search_threshold.yaml | 50 + .../suites/011_sift50m_acc.yaml | 19 + .../suites/add_flush_performance.yaml | 20 + tests/milvus_benchmark/suites/ann_debug.yaml | 26 + tests/milvus_benchmark/suites/clean.yaml | 24 + .../milvus_benchmark/suites/cpu_accuracy.yaml | 61 ++ .../suites/cpu_accuracy_ann.yaml | 66 +- .../suites/cpu_accuracy_ann_debug.yaml | 25 + .../suites/cpu_accuracy_ann_hnsw.yaml | 43 + .../suites/cpu_accuracy_ann_pq.yaml | 26 + .../suites/cpu_accuracy_nsg.yaml | 21 + .../suites/cpu_build_performance.yaml | 19 + .../suites/cpu_search_binary.yaml | 67 ++ .../cpu_search_performance_jaccard.yaml | 20 + .../suites/cpu_search_performance_sift1b.yaml | 22 + .../suites/cpu_search_performance_sift1m.yaml | 20 + .../cpu_search_performance_sift50m.yaml | 20 + .../suites/cpu_stability_sift50m.yaml | 27 + tests/milvus_benchmark/suites/debug.yaml | 90 ++ .../milvus_benchmark/suites/debug_build.yaml | 23 + .../suites/debug_gpu_search.yaml | 26 + .../suites/file_size_debug.yaml | 44 + .../milvus_benchmark/suites/gpu_accuracy.yaml | 41 + .../suites/gpu_accuracy_ann.yaml | 28 +- .../suites/gpu_accuracy_ann_debug.yaml | 25 + .../suites/gpu_accuracy_debug.yaml | 40 + .../suites/gpu_accuracy_sift1b.yaml | 59 ++ .../suites/gpu_accuracy_sift1m.yaml | 21 + .../gpu_build_performance_hamming50m.yaml | 36 + .../gpu_build_performance_jaccard50m.yaml | 20 + .../milvus_benchmark/suites/gpu_build_pq.yaml | 47 + .../suites/gpu_build_sift1b_sq8h.yaml | 20 + .../suites/gpu_search_performance.yaml | 247 +++++ .../gpu_search_performance_hamming50m.yaml | 22 + .../gpu_search_performance_jaccard50m.yaml | 22 + .../suites/gpu_search_performance_sift1b.yaml | 62 ++ .../suites/gpu_search_performance_sift1m.yaml | 22 + .../gpu_search_performance_sift50m.yaml | 146 +++ .../suites/gpu_search_stability.yaml | 23 + .../suites/gpu_stability_sift50m.yaml | 27 + .../suites/insert_binary.yaml | 39 + .../suites/insert_performance_deep1b.yaml | 87 ++ .../suites/locust_insert.yaml | 23 + tests/milvus_benchmark/suites/locust_mix.yaml | 53 ++ .../suites/locust_search.yaml | 34 + .../suites/loop_stability.yaml | 17 + tests/milvus_benchmark/suites/metric.yaml | 47 + tests/milvus_benchmark/suites/poseidon.yaml | 26 + tests/milvus_benchmark/suites/pq.yaml | 27 + tests/milvus_benchmark/suites/qps.yaml | 27 + .../milvus_benchmark/suites/search_debug.yaml | 92 ++ .../suites/shards_ann_debug.yaml | 25 + .../suites/shards_insert_performance.yaml | 17 + .../shards_insert_performance_sift1m.yaml | 19 + .../suites/shards_loop_stability.yaml | 16 + .../shards_search_performance_sift1m.yaml | 12 + tests/milvus_benchmark/task/task.py | 0 tests/milvus_benchmark/test_loop.py | 52 + tests/milvus_benchmark/utils.py | 453 +++------ 176 files changed, 8987 insertions(+), 1350 deletions(-) create mode 100644 tests/milvus_benchmark/ci/jenkinsfile/cleanupShards.groovy create mode 100644 tests/milvus_benchmark/ci/jenkinsfile/deploy_shards_test.groovy delete mode 100644 tests/milvus_benchmark/conf/log_config.conf delete mode 100644 tests/milvus_benchmark/conf/server_config.yaml delete mode 100644 tests/milvus_benchmark/conf/server_config.yaml.cpu delete mode 100644 tests/milvus_benchmark/conf/server_config.yaml.multi delete mode 100644 tests/milvus_benchmark/conf/server_config.yaml.single create mode 100644 tests/milvus_benchmark/docker_utils.py create mode 100644 tests/milvus_benchmark/executors/__init__.py create mode 100644 tests/milvus_benchmark/executors/shell.py create mode 100644 tests/milvus_benchmark/handlers/__init__.py create mode 100644 tests/milvus_benchmark/helm_utils.py create mode 100644 tests/milvus_benchmark/locust_file.py create mode 100644 tests/milvus_benchmark/locust_flush_task.py create mode 100644 tests/milvus_benchmark/locust_get_entity_task.py create mode 100644 tests/milvus_benchmark/locust_insert_task.py create mode 100644 tests/milvus_benchmark/locust_search_task.py create mode 100644 tests/milvus_benchmark/locust_task.py create mode 100644 tests/milvus_benchmark/locust_tasks.py create mode 100644 tests/milvus_benchmark/locust_user.py create mode 100644 tests/milvus_benchmark/mix_task.py delete mode 100644 tests/milvus_benchmark/report.py create mode 100644 tests/milvus_benchmark/results/__init__.py create mode 100644 tests/milvus_benchmark/results/reporter.py create mode 100644 tests/milvus_benchmark/runners/__init__.py create mode 100644 tests/milvus_benchmark/runners/locust_runner.py create mode 100644 tests/milvus_benchmark/scheduler/010_data.json create mode 100644 tests/milvus_benchmark/scheduler/011_data.json create mode 100644 tests/milvus_benchmark/scheduler/011_data_acc_debug.json rename tests/milvus_benchmark/scheduler/{ann_acc.json => 011_data_gpu_build.json} (69%) create mode 100644 tests/milvus_benchmark/scheduler/011_data_insert.json create mode 100644 tests/milvus_benchmark/scheduler/011_data_search_debug.json create mode 100644 tests/milvus_benchmark/scheduler/011_delete.json rename tests/milvus_benchmark/scheduler/{crud_flush.json => cqx.json} (74%) delete mode 100644 tests/milvus_benchmark/scheduler/crud_search.json rename tests/milvus_benchmark/scheduler/{crud_build.json => filter.json} (75%) rename tests/milvus_benchmark/scheduler/{crud_add.json => locust.json} (74%) rename tests/milvus_benchmark/scheduler/{hnsw.json => locust_mix_debug.json} (56%) rename tests/milvus_benchmark/scheduler/{crud_stability.json => loop.json} (61%) delete mode 100644 tests/milvus_benchmark/scheduler/search_performance.json create mode 100644 tests/milvus_benchmark/scheduler/shards.json create mode 100644 tests/milvus_benchmark/scheduler/shards_ann.json create mode 100644 tests/milvus_benchmark/scheduler/shards_debug.json create mode 100644 tests/milvus_benchmark/scheduler/shards_stability.json create mode 100644 tests/milvus_benchmark/scheduler/yx.json delete mode 100644 tests/milvus_benchmark/scripts/default_config.json delete mode 100644 tests/milvus_benchmark/scripts/scheduler.py create mode 100644 tests/milvus_benchmark/search_task.py create mode 100644 tests/milvus_benchmark/suites/011_add_flush_performance.yaml create mode 100644 tests/milvus_benchmark/suites/011_build_debug.yaml create mode 100644 tests/milvus_benchmark/suites/011_compact_performance.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_accuracy.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_accuracy_ann.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_accuracy_ann_debug.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_accuracy_rhnsw.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_build.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_build_binary.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_build_hnsw.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_build_rhnsw.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_search.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_search_binary.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_search_sift10m.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_search_sift10m_100k.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_search_sift10m_filter.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_search_sift10m_hnsw.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_search_sift10m_ivf.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_search_sift1b.yaml create mode 100644 tests/milvus_benchmark/suites/011_cpu_search_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/011_delete_performance.yaml create mode 100644 tests/milvus_benchmark/suites/011_get_id.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_accuracy.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_accuracy_ann.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_accuracy_ann_debug.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_accuracy_debug.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_build.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_build_debug.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_build_sift10m.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_build_sift1b.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_build_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_search.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_search_debug.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_search_sift10m.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_search_sift10m_100k.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_search_sift10m_filter.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_search_sift10m_ivf.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_search_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_sift50m_ivf.yaml create mode 100644 tests/milvus_benchmark/suites/011_gpu_stability.yaml create mode 100644 tests/milvus_benchmark/suites/011_insert_data.yaml create mode 100644 tests/milvus_benchmark/suites/011_insert_debug.yaml create mode 100644 tests/milvus_benchmark/suites/011_insert_performance.yaml create mode 100644 tests/milvus_benchmark/suites/011_locust_search.yaml create mode 100644 tests/milvus_benchmark/suites/011_search_dsl.yaml create mode 100644 tests/milvus_benchmark/suites/011_search_stability.yaml create mode 100644 tests/milvus_benchmark/suites/011_search_threshold.yaml create mode 100644 tests/milvus_benchmark/suites/011_sift50m_acc.yaml create mode 100644 tests/milvus_benchmark/suites/add_flush_performance.yaml create mode 100644 tests/milvus_benchmark/suites/ann_debug.yaml create mode 100644 tests/milvus_benchmark/suites/clean.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_accuracy.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_accuracy_ann_debug.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_accuracy_ann_hnsw.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_accuracy_ann_pq.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_accuracy_nsg.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_build_performance.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_search_binary.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_search_performance_jaccard.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_search_performance_sift1b.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_search_performance_sift1m.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_search_performance_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_stability_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/debug.yaml create mode 100644 tests/milvus_benchmark/suites/debug_build.yaml create mode 100644 tests/milvus_benchmark/suites/debug_gpu_search.yaml create mode 100644 tests/milvus_benchmark/suites/file_size_debug.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_accuracy.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_accuracy_ann_debug.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_accuracy_debug.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_accuracy_sift1b.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_accuracy_sift1m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_build_performance_hamming50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_build_performance_jaccard50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_build_pq.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_build_sift1b_sq8h.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance_hamming50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance_jaccard50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance_sift1b.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance_sift1m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_stability.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_stability_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/insert_binary.yaml create mode 100644 tests/milvus_benchmark/suites/insert_performance_deep1b.yaml create mode 100644 tests/milvus_benchmark/suites/locust_insert.yaml create mode 100644 tests/milvus_benchmark/suites/locust_mix.yaml create mode 100644 tests/milvus_benchmark/suites/locust_search.yaml create mode 100644 tests/milvus_benchmark/suites/loop_stability.yaml create mode 100644 tests/milvus_benchmark/suites/metric.yaml create mode 100644 tests/milvus_benchmark/suites/poseidon.yaml create mode 100644 tests/milvus_benchmark/suites/pq.yaml create mode 100644 tests/milvus_benchmark/suites/qps.yaml create mode 100644 tests/milvus_benchmark/suites/search_debug.yaml create mode 100644 tests/milvus_benchmark/suites/shards_ann_debug.yaml create mode 100644 tests/milvus_benchmark/suites/shards_insert_performance.yaml create mode 100644 tests/milvus_benchmark/suites/shards_insert_performance_sift1m.yaml create mode 100644 tests/milvus_benchmark/suites/shards_loop_stability.yaml create mode 100644 tests/milvus_benchmark/suites/shards_search_performance_sift1m.yaml create mode 100644 tests/milvus_benchmark/task/task.py create mode 100644 tests/milvus_benchmark/test_loop.py diff --git a/tests/milvus_benchmark/.gitignore b/tests/milvus_benchmark/.gitignore index 3688f39a9a..005dce9907 100644 --- a/tests/milvus_benchmark/.gitignore +++ b/tests/milvus_benchmark/.gitignore @@ -6,5 +6,7 @@ logs/ __pycache__/ venv .idea +nohup.out -suites/ +*.swp +*.swo diff --git a/tests/milvus_benchmark/ci/jenkinsfile/cleanup.groovy b/tests/milvus_benchmark/ci/jenkinsfile/cleanup.groovy index 622c4d3ff6..ec38b34618 100644 --- a/tests/milvus_benchmark/ci/jenkinsfile/cleanup.groovy +++ b/tests/milvus_benchmark/ci/jenkinsfile/cleanup.groovy @@ -1,12 +1,12 @@ try { - def result = sh script: "helm status benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + def result = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true if (!result) { - sh "helm del --purge benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}" + sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}" } } catch (exc) { - def result = sh script: "helm status benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + def result = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true if (!result) { - sh "helm del --purge benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}" + sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}" } throw exc } diff --git a/tests/milvus_benchmark/ci/jenkinsfile/cleanupShards.groovy b/tests/milvus_benchmark/ci/jenkinsfile/cleanupShards.groovy new file mode 100644 index 0000000000..ac0f4a4336 --- /dev/null +++ b/tests/milvus_benchmark/ci/jenkinsfile/cleanupShards.groovy @@ -0,0 +1,13 @@ +try { + def result = sh script: "helm status -n milvus ${env.HELM_SHARDS_RELEASE_NAME}", returnStatus: true + if (!result) { + sh "helm uninstall -n milvus ${env.HELM_SHARDS_RELEASE_NAME}" + } +} catch (exc) { + def result = sh script: "helm status -n milvus ${env.HELM_SHARDS_RELEASE_NAME}", returnStatus: true + if (!result) { + sh "helm uninstall -n milvus ${env.HELM_SHARDS_RELEASE_NAME}" + } + throw exc +} + diff --git a/tests/milvus_benchmark/ci/jenkinsfile/deploy_shards_test.groovy b/tests/milvus_benchmark/ci/jenkinsfile/deploy_shards_test.groovy new file mode 100644 index 0000000000..2a4c88e91f --- /dev/null +++ b/tests/milvus_benchmark/ci/jenkinsfile/deploy_shards_test.groovy @@ -0,0 +1,21 @@ +timeout(time: 12, unit: 'HOURS') { + try { + dir ("milvus-helm") { + // sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + // sh 'helm repo update' + checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]]) + } + dir ("milvus_benchmark") { + print "Git clone url: ${TEST_URL}:${TEST_BRANCH}" + checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]]) + print "Install requirements" + // sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com" + sh "python3 -m pip install -r requirements.txt" + sh "python3 -m pip install git+${TEST_LIB_URL}" + sh "python3 main.py --image-version=${params.IMAGE_VERSION} --schedule-conf=scheduler/${params.SHARDS_CONFIG_FILE} --deploy-mode=${params.DEPLOY_MODE}" + } + } catch (exc) { + echo 'Deploy SHARDS Test Failed !' + throw exc + } +} diff --git a/tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy b/tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy index f31e2215b0..8941089cd7 100644 --- a/tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy +++ b/tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy @@ -1,21 +1,19 @@ -timeout(time: 4000, unit: 'MINUTES') { - try { - dir ("milvus-helm") { - // sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' - // sh 'helm repo update' - checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]]) - } - dir ("milvus_benchmark") { - print "Git clone url: ${TEST_URL}:${TEST_BRANCH}" - checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]]) - print "Install requirements" - // sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com" - sh "python3 -m pip install -r requirements.txt" - sh "python3 -m pip install git+${TEST_LIB_URL}" - sh "python3 main.py --image-version=${params.IMAGE_VERSION} --schedule-conf=scheduler/${params.CONFIG_FILE}" - } - } catch (exc) { - echo 'Deploy Test Failed !' - throw exc +try { + dir ("milvus-helm") { + // sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + // sh 'helm repo update' + checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]]) } + dir ("milvus_benchmark") { + print "Git clone url: ${TEST_URL}:${TEST_BRANCH}" + checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]]) + print "Install requirements" + // sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com" + sh "python3 -m pip install -r requirements.txt" + sh "python3 -m pip install git+${TEST_LIB_URL}" + sh "python3 main.py --image-version=${params.IMAGE_VERSION} --schedule-conf=scheduler/${params.CONFIG_FILE} --deploy-mode=${params.DEPLOY_MODE}" + } +} catch (exc) { + echo 'Deploy Test Failed !' + throw exc } diff --git a/tests/milvus_benchmark/ci/jenkinsfile/publishDailyImages.groovy b/tests/milvus_benchmark/ci/jenkinsfile/publishDailyImages.groovy index 07214a95f8..54ac5da9c4 100644 --- a/tests/milvus_benchmark/ci/jenkinsfile/publishDailyImages.groovy +++ b/tests/milvus_benchmark/ci/jenkinsfile/publishDailyImages.groovy @@ -1,4 +1,4 @@ -timeout(time: 15, unit: 'MINUTES') { +timeout(time: 30, unit: 'MINUTES') { def imageName = "milvus/engine:${DOCKER_VERSION}" def remoteImageName = "milvusdb/daily-build:${REMOTE_DOCKER_VERSION}" def localDockerRegistryImage = "${params.LOCAL_DOKCER_REGISTRY_URL}/${imageName}" diff --git a/tests/milvus_benchmark/ci/main_jenkinsfile b/tests/milvus_benchmark/ci/main_jenkinsfile index 6431b2f09c..ca0130e3f5 100644 --- a/tests/milvus_benchmark/ci/main_jenkinsfile +++ b/tests/milvus_benchmark/ci/main_jenkinsfile @@ -6,17 +6,21 @@ pipeline { } parameters{ - string defaultValue: 'master', description: 'server image version', name: 'IMAGE_VERSION', trim: true - string defaultValue: '080_data.json', description: 'test suite config yaml', name: 'CONFIG_FILE', trim: true + string defaultValue: '0.11.1', description: 'server image version', name: 'IMAGE_VERSION', trim: true + choice choices: ['single', 'shards'], description: 'server deploy mode', name: 'DEPLOY_MODE' + string defaultValue: '011_data.json', description: 'test suite config yaml', name: 'CONFIG_FILE', trim: true + string defaultValue: 'shards.json', description: 'shards test suite config yaml', name: 'SHARDS_CONFIG_FILE', trim: true string defaultValue: '09509e53-9125-4f5d-9ce8-42855987ad67', description: 'git credentials', name: 'GIT_USER', trim: true } environment { HELM_URL = "https://github.com/milvus-io/milvus-helm.git" - HELM_BRANCH = "master" + HELM_BRANCH = "0.11.0" TEST_URL = "git@192.168.1.105:Test/milvus_benchmark.git" - TEST_BRANCH = "master" + TEST_BRANCH = "0.11.1" TEST_LIB_URL = "http://192.168.1.105:6060/Test/milvus_metrics.git" + HELM_RELEASE_NAME = "milvus-benchmark-test-${env.BUILD_NUMBER}" + HELM_SHARDS_RELEASE_NAME = "milvus-shards-benchmark-test-${env.BUILD_NUMBER}" } stages { @@ -35,7 +39,7 @@ pipeline { spec: containers: - name: milvus-test-env - image: registry.zilliz.com/milvus/milvus-test-env:v0.2 + image: registry.zilliz.com/milvus/milvus-test-env:v0.3 command: - cat tty: true @@ -93,11 +97,13 @@ pipeline { stage("Deploy Test") { steps { - gitlabCommitStatus(name: 'Deploy Test') { - container('milvus-test-env') { - script { - print "In Deploy Test Stage" + container('milvus-test-env') { + script { + print "In Deploy Test Stage" + if ("${params.DEPLOY_MODE}" == "single") { load "${env.WORKSPACE}/ci/jenkinsfile/deploy_test.groovy" + } else { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy_shards_test.groovy" } } } @@ -106,10 +112,12 @@ pipeline { stage ("Cleanup Env") { steps { - gitlabCommitStatus(name: 'Cleanup Env') { - container('milvus-test-env') { - script { + container('milvus-test-env') { + script { + if ("${params.DEPLOY_MODE}" == "single") { load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy" + } else { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanupShards.groovy" } } } @@ -117,13 +125,6 @@ pipeline { } } post { - always { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy" - } - } - } success { script { echo "Milvus benchmark test success !" diff --git a/tests/milvus_benchmark/ci/publish_jenkinsfile b/tests/milvus_benchmark/ci/publish_jenkinsfile index 353ed82f9d..9936030c0e 100644 --- a/tests/milvus_benchmark/ci/publish_jenkinsfile +++ b/tests/milvus_benchmark/ci/publish_jenkinsfile @@ -9,6 +9,7 @@ pipeline { string defaultValue: 'registry.zilliz.com', description: 'Local Docker registry URL', name: 'LOCAL_DOKCER_REGISTRY_URL', trim: true string defaultValue: 'registry-1.docker.io', description: 'Remote Docker registry URL', name: 'REMOTE_DOKCER_REGISTRY_URL', trim: true string defaultValue: 'milvus-docker-access-token', description: 'Remote Docker credentials id', name: 'REMOTE_DOCKER_CREDENTIALS_ID', trim: true + string(defaultValue: "master", description: 'Milvus server version', name: 'BRANCH') } environment { @@ -41,9 +42,9 @@ pipeline { stages { stage("Publish Docker Images") { environment { - DOCKER_VERSION = "master-${BINARY_VERSION}-${OS_NAME}-release" - REMOTE_DOCKER_VERSION = "${OS_NAME}-${BINARY_VERSION}-${DAILY_BUILD_VERSION}" - REMOTE_DOCKER_LATEST_VERSION = "${OS_NAME}-${BINARY_VERSION}-latest" + DOCKER_VERSION = "${params.BRANCH}-${BINARY_VERSION}-${OS_NAME}-release" + REMOTE_DOCKER_VERSION = "${params.BRANCH}-${OS_NAME}-${BINARY_VERSION}-${DAILY_BUILD_VERSION}" + REMOTE_DOCKER_LATEST_VERSION = "${params.BRANCH}-${OS_NAME}-${BINARY_VERSION}-latest" } agent { diff --git a/tests/milvus_benchmark/client.py b/tests/milvus_benchmark/client.py index b6670bb6bd..0d4f8e52b6 100644 --- a/tests/milvus_benchmark/client.py +++ b/tests/milvus_benchmark/client.py @@ -5,64 +5,76 @@ import logging import json import time, datetime from multiprocessing import Process -from milvus import Milvus, IndexType, MetricType +from milvus import Milvus, DataType +import numpy as np +import utils logger = logging.getLogger("milvus_benchmark.client") SERVER_HOST_DEFAULT = "127.0.0.1" -# SERVER_HOST_DEFAULT = "192.168.1.130" SERVER_PORT_DEFAULT = 19530 INDEX_MAP = { - "flat": IndexType.FLAT, - "ivf_flat": IndexType.IVFLAT, - "ivf_sq8": IndexType.IVF_SQ8, - "nsg": IndexType.RNSG, - "ivf_sq8h": IndexType.IVF_SQ8H, - "ivf_pq": IndexType.IVF_PQ, - "hnsw": IndexType.HNSW, - "annoy": IndexType.ANNOY + "flat": "FLAT", + "ivf_flat": "IVF_FLAT", + "ivf_sq8": "IVF_SQ8", + "nsg": "NSG", + "ivf_sq8h": "IVF_SQ8_HYBRID", + "ivf_pq": "IVF_PQ", + "hnsw": "HNSW", + "annoy": "ANNOY", + "bin_flat": "BIN_FLAT", + "bin_ivf_flat": "BIN_IVF_FLAT", + "rhnsw_pq": "RHNSW_PQ", + "rhnsw_sq": "RHNSW_SQ" } epsilon = 0.1 + def time_wrapper(func): """ This decorator prints the execution time for the decorated function. """ + def wrapper(*args, **kwargs): start = time.time() + logger.info("Milvus {} start".format(func.__name__)) result = func(*args, **kwargs) end = time.time() logger.info("Milvus {} run in {}s".format(func.__name__, round(end - start, 2))) return result + return wrapper class MilvusClient(object): - def __init__(self, collection_name=None, ip=None, port=None, timeout=60): + def __init__(self, collection_name=None, host=None, port=None, timeout=180): self._collection_name = collection_name - try: - i = 1 - start_time = time.time() - if not ip: + start_time = time.time() + if not host: + host = SERVER_HOST_DEFAULT + if not port: + port = SERVER_PORT_DEFAULT + logger.debug(host) + logger.debug(port) + # retry connect remote server + i = 0 + while time.time() < start_time + timeout: + try: self._milvus = Milvus( - host = SERVER_HOST_DEFAULT, - port = SERVER_PORT_DEFAULT) - else: - # retry connect for remote server - while time.time() < start_time + timeout: - try: - self._milvus = Milvus( - host = ip, - port = port) - if self._milvus.server_status(): - logger.debug("Try connect times: %d, %s" % (i, round(time.time() - start_time, 2))) - break - except Exception as e: - logger.debug("Milvus connect failed") - i = i + 1 + host=host, + port=port, + try_connect=False, + pre_ping=False) + break + except Exception as e: + logger.error(str(e)) + logger.error("Milvus connect failed: %d times" % i) + i = i + 1 + time.sleep(i) - except Exception as e: - raise e + if time.time() > start_time + timeout: + raise Exception("Server connect timeout") + # self._metric_type = None def __str__(self): return 'Milvus collection %s' % self._collection_name @@ -70,6 +82,8 @@ class MilvusClient(object): def check_status(self, status): if not status.OK(): logger.error(status.message) + logger.error(self._milvus.server_status()) + logger.error(self.count()) raise Exception("Status not ok") def check_result_ids(self, result): @@ -79,126 +93,281 @@ class MilvusClient(object): logger.error(item[0].distance) raise Exception("Distance wrong") - def create_collection(self, collection_name, dimension, index_file_size, metric_type): - if not self._collection_name: - self._collection_name = collection_name - if metric_type == "l2": - metric_type = MetricType.L2 - elif metric_type == "ip": - metric_type = MetricType.IP - elif metric_type == "jaccard": - metric_type = MetricType.JACCARD - elif metric_type == "hamming": - metric_type = MetricType.HAMMING - elif metric_type == "sub": - metric_type = MetricType.SUBSTRUCTURE - elif metric_type == "super": - metric_type = MetricType.SUPERSTRUCTURE - else: - logger.error("Not supported metric_type: %s" % metric_type) - create_param = {'collection_name': collection_name, - 'dimension': dimension, - 'index_file_size': index_file_size, - "metric_type": metric_type} - status = self._milvus.create_collection(create_param) + # only support the given field name + def create_collection(self, dimension, segment_row_limit, data_type=DataType.FLOAT_VECTOR, auto_id=False, + collection_name=None, other_fields=None): + self._dimension = dimension + if not collection_name: + collection_name = self._collection_name + vec_field_name = utils.get_default_field_name(data_type) + fields = [{"name": vec_field_name, "type": data_type, "params": {"dim": dimension}}] + if other_fields: + other_fields = other_fields.split(",") + if "int" in other_fields: + fields.append({"name": utils.DEFAULT_INT_FIELD_NAME, "type": DataType.INT64}) + if "float" in other_fields: + fields.append({"name": utils.DEFAULT_FLOAT_FIELD_NAME, "type": DataType.FLOAT}) + create_param = { + "fields": fields, + "segment_row_limit": segment_row_limit, + "auto_id": auto_id} + try: + self._milvus.create_collection(collection_name, create_param) + logger.info("Create collection: <%s> successfully" % collection_name) + except Exception as e: + logger.error(str(e)) + raise + + def generate_values(self, data_type, vectors, ids): + values = None + if data_type in [DataType.INT32, DataType.INT64]: + values = ids + elif data_type in [DataType.FLOAT, DataType.DOUBLE]: + values = [(i + 0.0) for i in ids] + elif data_type in [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR]: + values = vectors + return values + + def generate_entities(self, vectors, ids=None, collection_name=None): + entities = [] + if collection_name is None: + collection_name = self._collection_name + info = self.get_info(collection_name) + for field in info["fields"]: + field_type = field["type"] + entities.append( + {"name": field["name"], "type": field_type, "values": self.generate_values(field_type, vectors, ids)}) + return entities + + @time_wrapper + def insert(self, entities, ids=None, collection_name=None): + tmp_collection_name = self._collection_name if collection_name is None else collection_name + insert_ids = self._milvus.insert(tmp_collection_name, entities, ids=ids) + return insert_ids + + def insert_rand(self): + insert_xb = random.randint(1, 100) + X = [[random.random() for _ in range(self._dimension)] for _ in range(insert_xb)] + ids = [] + for i in range(insert_xb): + ids.append(random.randint(1, 10000000)) + entities = self.generate_entities(X, ids) + count_before = self.count() + res_ids = self.insert(entities, ids=ids) + assert res_ids == ids + self.flush() + assert count_before + insert_xb == self.count() + + def get_rand_ids(self, length): + segment_ids = [] + while True: + stats = self.get_stats() + segments = stats["partitions"][0]["segments"] + # random choice one segment + segment = random.choice(segments) + try: + segment_ids = self._milvus.list_id_in_segment(self._collection_name, segment["id"]) + except Exception as e: + logger.error(str(e)) + if not len(segment_ids): + continue + elif len(segment_ids) > length: + return random.sample(segment_ids, length) + else: + logger.debug("Reset length: %d" % len(segment_ids)) + return segment_ids + + # def get_rand_ids_each_segment(self, length): + # res = [] + # status, stats = self._milvus.get_collection_stats(self._collection_name) + # self.check_status(status) + # segments = stats["partitions"][0]["segments"] + # segments_num = len(segments) + # # random choice from each segment + # for segment in segments: + # status, segment_ids = self._milvus.list_id_in_segment(self._collection_name, segment["name"]) + # self.check_status(status) + # res.extend(segment_ids[:length]) + # return segments_num, res + + # def get_rand_entities(self, length): + # ids = self.get_rand_ids(length) + # status, get_res = self._milvus.get_entity_by_id(self._collection_name, ids) + # self.check_status(status) + # return ids, get_res + + @time_wrapper + def get_entities(self, get_ids): + get_res = self._milvus.get_entity_by_id(self._collection_name, get_ids) + return get_res + + @time_wrapper + def delete(self, ids, collection_name=None): + tmp_collection_name = self._collection_name if collection_name is None else collection_name + self._milvus.delete_entity_by_id(tmp_collection_name, ids) + + def delete_rand(self): + delete_id_length = random.randint(1, 100) + count_before = self.count() + logger.info("%s: length to delete: %d" % (self._collection_name, delete_id_length)) + delete_ids = self.get_rand_ids(delete_id_length) + self.delete(delete_ids) + self.flush() + logger.info("%s: count after delete: %d" % (self._collection_name, self.count())) + get_res = self._milvus.get_entity_by_id(self._collection_name, delete_ids) + for item in get_res: + assert not item + # if count_before - len(delete_ids) < self.count(): + # logger.error(delete_ids) + # raise Exception("Error occured") + + @time_wrapper + def flush(self, collection_name=None): + tmp_collection_name = self._collection_name if collection_name is None else collection_name + self._milvus.flush([tmp_collection_name]) + + @time_wrapper + def compact(self, collection_name=None): + tmp_collection_name = self._collection_name if collection_name is None else collection_name + status = self._milvus.compact(tmp_collection_name) self.check_status(status) @time_wrapper - def insert(self, X, ids=None): - status, result = self._milvus.add_vectors(self._collection_name, X, ids) - self.check_status(status) - return status, result - - @time_wrapper - def delete_vectors(self, ids): - status = self._milvus.delete_by_id(self._collection_name, ids) - self.check_status(status) - - @time_wrapper - def flush(self): - status = self._milvus.flush([self._collection_name]) - self.check_status(status) - - @time_wrapper - def compact(self): - status = self._milvus.compact(self._collection_name) - self.check_status(status) - - @time_wrapper - def create_index(self, index_type, index_param=None): + def create_index(self, field_name, index_type, metric_type, index_param=None): index_type = INDEX_MAP[index_type] - logger.info("Building index start, collection_name: %s, index_type: %s" % (self._collection_name, index_type)) + metric_type = utils.metric_type_trans(metric_type) + logger.info("Building index start, collection_name: %s, index_type: %s, metric_type: %s" % ( + self._collection_name, index_type, metric_type)) if index_param: logger.info(index_param) - status = self._milvus.create_index(self._collection_name, index_type, index_param) - self.check_status(status) + index_params = { + "index_type": index_type, + "metric_type": metric_type, + "params": index_param + } + self._milvus.create_index(self._collection_name, field_name, index_params) def describe_index(self): - status, result = self._milvus.describe_index(self._collection_name) - self.check_status(status) - index_type = None - for k, v in INDEX_MAP.items(): - if result._index_type == v: - index_type = k - break - return {"index_type": index_type, "index_param": result._params} + # stats = self.get_stats() + info = self.get_info() + index_info = {"index_type": "flat", "index_param": None} + for field in info["fields"]: + for index in field['indexes']: + if not index or "index_type" not in index: + continue + else: + for k, v in INDEX_MAP.items(): + if index['index_type'] == v: + index_info['index_type'] = k + index_info['index_param'] = index['params'] + return index_info + return index_info - def drop_index(self): + def drop_index(self, field_name): logger.info("Drop index: %s" % self._collection_name) - return self._milvus.drop_index(self._collection_name) + return self._milvus.drop_index(self._collection_name, field_name) @time_wrapper - def query(self, X, top_k, search_param=None): - status, result = self._milvus.search_vectors(self._collection_name, top_k, query_records=X, params=search_param) - self.check_status(status) + def query(self, vector_query, filter_query=None, collection_name=None): + tmp_collection_name = self._collection_name if collection_name is None else collection_name + must_params = [vector_query] + if filter_query: + must_params.extend(filter_query) + query = { + "bool": {"must": must_params} + } + result = self._milvus.search(tmp_collection_name, query) + # logging.info(len(result)) + # logging.info(len(result[0])) return result - @time_wrapper - def query_ids(self, top_k, ids, search_param=None): - status, result = self._milvus.search_by_ids(self._collection_name, ids, top_k, params=search_param) - self.check_result_ids(result) - return result + def get_ids(self, result): + idss = result._entities.ids + ids = [] + len_idss = len(idss) + len_r = len(result) + top_k = len_idss // len_r + for offset in range(0, len_idss, top_k): + ids.append(idss[offset: min(offset + top_k, len_idss)]) + return ids - def count(self): - return self._milvus.count_collection(self._collection_name)[1] + def query_rand(self, nq_max=100): + # for ivf search + dimension = 128 + top_k = random.randint(1, 100) + nq = random.randint(1, nq_max) + nprobe = random.randint(1, 100) + search_param = {"nprobe": nprobe} + query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)] + metric_type = random.choice(["l2", "ip"]) + logger.info("%s, Search nq: %d, top_k: %d, nprobe: %d" % (self._collection_name, nq, top_k, nprobe)) + vec_field_name = utils.get_default_field_name() + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors, + "metric_type": utils.metric_type_trans(metric_type), + "params": search_param} + }} + self.query(vector_query) - def delete(self, timeout=120): + def count(self, collection_name=None): + if collection_name is None: + collection_name = self._collection_name + row_count = self._milvus.count_entities(collection_name) + logger.debug("Row count: %d in collection: <%s>" % (row_count, collection_name)) + return row_count + + def drop(self, timeout=120, collection_name=None): timeout = int(timeout) - logger.info("Start delete collection: %s" % self._collection_name) - self._milvus.drop_collection(self._collection_name) + if collection_name is None: + collection_name = self._collection_name + logger.info("Start delete collection: %s" % collection_name) + self._milvus.drop_collection(collection_name) i = 0 while i < timeout: - if self.count(): - time.sleep(1) - i = i + 1 - continue - else: + try: + row_count = self.count(collection_name=collection_name) + if row_count: + time.sleep(1) + i = i + 1 + continue + else: + break + except Exception as e: + logger.debug(str(e)) break if i >= timeout: logger.error("Delete collection timeout") - def describe(self): - return self._milvus.describe_collection(self._collection_name) + def get_stats(self): + return self._milvus.get_collection_stats(self._collection_name) + + def get_info(self, collection_name=None): + # pdb.set_trace() + if collection_name is None: + collection_name = self._collection_name + return self._milvus.get_collection_info(collection_name) def show_collections(self): - return self._milvus.show_collections() + return self._milvus.list_collections() def exists_collection(self, collection_name=None): if collection_name is None: collection_name = self._collection_name - status, res = self._milvus.has_collection(collection_name) - # self.check_status(status) + res = self._milvus.has_collection(collection_name) return res + def clean_db(self): + collection_names = self.show_collections() + for name in collection_names: + self.drop(collection_name=name) + @time_wrapper def preload_collection(self): - status = self._milvus.preload_collection(self._collection_name, timeout=3000) - self.check_status(status) - return status + return self._milvus.load_collection(self._collection_name, timeout=3000) def get_server_version(self): - status, res = self._milvus.server_version() - return res + return self._milvus.server_version() def get_server_mode(self): return self.cmd("mode") @@ -207,166 +376,26 @@ class MilvusClient(object): return self.cmd("build_commit_id") def get_server_config(self): - return json.loads(self.cmd("get_config *")) + return json.loads(self.cmd("get_milvus_config")) def get_mem_info(self): result = json.loads(self.cmd("get_system_info")) result_human = { # unit: Gb - "memory_used": round(int(result["memory_used"]) / (1024*1024*1024), 2) + "memory_used": round(int(result["memory_used"]) / (1024 * 1024 * 1024), 2) } return result_human def cmd(self, command): - status, res = self._milvus._cmd(command) + res = self._milvus._cmd(command) logger.info("Server command: %s, result: %s" % (command, res)) - self.check_status(status) return res + @time_wrapper + def set_config(self, parent_key, child_key, value): + self._milvus.set_config(parent_key, child_key, value) -def fit(collection_name, X): - milvus = Milvus() - milvus.connect(host = SERVER_HOST_DEFAULT, port = SERVER_PORT_DEFAULT) - start = time.time() - status, ids = milvus.add_vectors(collection_name, X) - end = time.time() - logger(status, round(end - start, 2)) - - -def fit_concurrent(collection_name, process_num, vectors): - processes = [] - - for i in range(process_num): - p = Process(target=fit, args=(collection_name, vectors, )) - processes.append(p) - p.start() - for p in processes: - p.join() - - -if __name__ == "__main__": - import numpy - import sklearn.preprocessing - - # collection_name = "tset_test" - # # collection_name = "test_tset1" - # m = MilvusClient(collection_name) - # m.delete() - # time.sleep(2) - # m.create_collection(collection_name, 128, 20, "ip") - - # print(m.describe()) - # print(m.count()) - # print(m.describe_index()) - # # sys.exit() - # tmp = [[random.random() for _ in range(128)] for _ in range(20000)] - # tmp1 = sklearn.preprocessing.normalize(tmp, axis=1, norm='l2') - # print(tmp1[0][0]) - # tmp = [[random.random() for _ in range(128)] for _ in range(20000)] - # tmp /= numpy.linalg.norm(tmp) - # print(tmp[0][0]) - - # sum_1 = 0 - # sum_2 = 0 - # for item in tmp: - # for i in item: - # sum_2 = sum_2 + i * i - # break - # for item in tmp1: - # for i in item: - # sum_1 = sum_1 + i * i - # break - # print(sum_1, sum_2) - # insert_vectors = tmp.tolist() - # # print(insert_vectors) - # for i in range(2): - # m.insert(insert_vectors) - - # time.sleep(5) - # print(m.create_index("ivf_flat", 16384)) - # X = [insert_vectors[0], insert_vectors[1], insert_vectors[2]] - # top_k = 5 - # nprobe = 1 - # print(m.query(X, top_k, nprobe)) - - # # print(m.drop_index()) - # print(m.describe_index()) - # sys.exit() - # # insert_vectors = [[random.random() for _ in range(128)] for _ in range(100000)] - # # for i in range(100): - # # m.insert(insert_vectors) - # # time.sleep(5) - # # print(m.describe_index()) - # # print(m.drop_index()) - # m.create_index("ivf_sq8h", 16384) - # print(m.count()) - # print(m.describe_index()) - - - - # sys.exit() - # print(m.create_index("ivf_sq8h", 16384)) - # print(m.count()) - # print(m.describe_index()) - import numpy as np - - # def mmap_fvecs(fname): - # x = np.memmap(fname, dtype='int32', mode='r') - # d = x[0] - # return x.view('float32').reshape(-1, d + 1)[:, 1:] - - # print(mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs")) - # SIFT_SRC_QUERY_DATA_DIR = '/poc/yuncong/ann_1000m' - # file_name = SIFT_SRC_QUERY_DATA_DIR+'/'+'query.npy' - # data = numpy.load(file_name) - # query_vectors = data[0:2].tolist() - # print(len(query_vectors)) - # results = m.query(query_vectors, 10, 10) - # result_ids = [] - # for result in results[1]: - # tmp = [] - # for item in result: - # tmp.append(item.id) - # result_ids.append(tmp) - # print(result_ids[0][:10]) - # # gt - # file_name = SIFT_SRC_QUERY_DATA_DIR+"/gnd/"+"idx_1M.ivecs" - # a = numpy.fromfile(file_name, dtype='int32') - # d = a[0] - # true_ids = a.reshape(-1, d + 1)[:, 1:].copy() - # print(true_ids[:3, :2]) - - # print(len(true_ids[0])) - # import numpy as np - # import sklearn.preprocessing - - # def mmap_fvecs(fname): - # x = np.memmap(fname, dtype='int32', mode='r') - # d = x[0] - # return x.view('float32').reshape(-1, d + 1)[:, 1:] - - # data = mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs") - # data = sklearn.preprocessing.normalize(data, axis=1, norm='l2') - # np.save("/test/milvus/deep1b/query.npy", data) - dimension = 4096 - insert_xb = 10000 - insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] - data = sklearn.preprocessing.normalize(insert_vectors, axis=1, norm='l2') - np.save("/test/milvus/raw_data/random/query_%d.npy" % dimension, data) - sys.exit() - - total_size = 100000000 - # total_size = 1000000000 - file_size = 100000 - # file_size = 100000 - dimension = 4096 - file_num = total_size // file_size - for i in range(file_num): - print(i) - # fname = "/test/milvus/raw_data/deep1b/binary_96_%05d" % i - fname = "/test/milvus/raw_data/random/binary_%dd_%05d" % (dimension, i) - # print(fname, i*file_size, (i+1)*file_size) - # single_data = data[i*file_size : (i+1)*file_size] - single_data = [[random.random() for _ in range(dimension)] for _ in range(file_size)] - single_data = sklearn.preprocessing.normalize(single_data, axis=1, norm='l2') - np.save(fname, single_data) + def create_partition(self, tag, collection_name=None): + if not collection_name: + collection_name = self._collection_name + self._milvus.create_partition(collection_name, tag) diff --git a/tests/milvus_benchmark/conf/log_config.conf b/tests/milvus_benchmark/conf/log_config.conf deleted file mode 100644 index c9c14d57f4..0000000000 --- a/tests/milvus_benchmark/conf/log_config.conf +++ /dev/null @@ -1,28 +0,0 @@ -* GLOBAL: - FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-global.log" - ENABLED = true - TO_FILE = true - TO_STANDARD_OUTPUT = false - SUBSECOND_PRECISION = 3 - PERFORMANCE_TRACKING = false - MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB -* DEBUG: - FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-debug.log" - ENABLED = true -* WARNING: - FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-warning.log" -* TRACE: - FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-trace.log" -* VERBOSE: - FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" - TO_FILE = false - TO_STANDARD_OUTPUT = false -## Error logs -* ERROR: - ENABLED = true - FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-error.log" -* FATAL: - ENABLED = true - FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" - diff --git a/tests/milvus_benchmark/conf/server_config.yaml b/tests/milvus_benchmark/conf/server_config.yaml deleted file mode 100644 index 46cb3a597c..0000000000 --- a/tests/milvus_benchmark/conf/server_config.yaml +++ /dev/null @@ -1,34 +0,0 @@ -server_config: - address: 0.0.0.0 - port: 19530 - deploy_mode: single - time_zone: UTC+8 - -db_config: - primary_path: /var/lib/milvus - secondary_path: - backend_url: sqlite://:@:/ - insert_buffer_size: 4 - preload_collection: - -metric_config: - enable_monitor: true - collector: prometheus - prometheus_config: - port: 8080 - -cache_config: - cpu_cache_capacity: 16 - cache_insert_data: false - -engine_config: - use_blas_threshold: 1100 - gpu_search_threshold: 1000 -gpu_resource_config: - enable: true - cache_capacity: 4 - search_resources: - - gpu0 - build_index_resources: - - gpu0 - diff --git a/tests/milvus_benchmark/conf/server_config.yaml.cpu b/tests/milvus_benchmark/conf/server_config.yaml.cpu deleted file mode 100644 index 2fb5cba83c..0000000000 --- a/tests/milvus_benchmark/conf/server_config.yaml.cpu +++ /dev/null @@ -1,31 +0,0 @@ -server_config: - address: 0.0.0.0 - port: 19530 - deploy_mode: single - time_zone: UTC+8 - -db_config: - primary_path: /opt/milvus - secondary_path: - backend_url: sqlite://:@:/ - insert_buffer_size: 4 - build_index_gpu: 0 - preload_collection: - -metric_config: - enable_monitor: false - collector: prometheus - prometheus_config: - port: 8080 - -cache_config: - cpu_cache_capacity: 16 - cpu_cache_threshold: 0.85 - cache_insert_data: false - -engine_config: - use_blas_threshold: 20 - -resource_config: - resource_pool: - - cpu \ No newline at end of file diff --git a/tests/milvus_benchmark/conf/server_config.yaml.multi b/tests/milvus_benchmark/conf/server_config.yaml.multi deleted file mode 100644 index dbef6a305b..0000000000 --- a/tests/milvus_benchmark/conf/server_config.yaml.multi +++ /dev/null @@ -1,33 +0,0 @@ -server_config: - address: 0.0.0.0 - port: 19530 - deploy_mode: single - time_zone: UTC+8 - -db_config: - primary_path: /opt/milvus - secondary_path: - backend_url: sqlite://:@:/ - insert_buffer_size: 4 - build_index_gpu: 0 - preload_collection: - -metric_config: - enable_monitor: false - collector: prometheus - prometheus_config: - port: 8080 - -cache_config: - cpu_cache_capacity: 16 - cpu_cache_threshold: 0.85 - cache_insert_data: false - -engine_config: - use_blas_threshold: 20 - -resource_config: - resource_pool: - - cpu - - gpu0 - - gpu1 \ No newline at end of file diff --git a/tests/milvus_benchmark/conf/server_config.yaml.single b/tests/milvus_benchmark/conf/server_config.yaml.single deleted file mode 100644 index 93b2bc2a6c..0000000000 --- a/tests/milvus_benchmark/conf/server_config.yaml.single +++ /dev/null @@ -1,32 +0,0 @@ -server_config: - address: 0.0.0.0 - port: 19530 - deploy_mode: single - time_zone: UTC+8 - -db_config: - primary_path: /opt/milvus - secondary_path: - backend_url: sqlite://:@:/ - insert_buffer_size: 4 - build_index_gpu: 0 - preload_collection: - -metric_config: - enable_monitor: false - collector: prometheus - prometheus_config: - port: 8080 - -cache_config: - cpu_cache_capacity: 16 - cpu_cache_threshold: 0.85 - cache_insert_data: false - -engine_config: - use_blas_threshold: 20 - -resource_config: - resource_pool: - - cpu - - gpu0 \ No newline at end of file diff --git a/tests/milvus_benchmark/docker_utils.py b/tests/milvus_benchmark/docker_utils.py new file mode 100644 index 0000000000..11b7d86bb6 --- /dev/null +++ b/tests/milvus_benchmark/docker_utils.py @@ -0,0 +1,125 @@ +# def pull_image(image): +# registry = image.split(":")[0] +# image_tag = image.split(":")[1] +# client = docker.APIClient(base_url='unix://var/run/docker.sock') +# logger.info("Start pulling image: %s" % image) +# return client.pull(registry, image_tag) + + +# def run_server(image, mem_limit=None, timeout=30, test_type="local", volume_name=None, db_slave=None): +# import colors + +# client = docker.from_env() +# # if mem_limit is None: +# # mem_limit = psutil.virtual_memory().available +# # logger.info('Memory limit:', mem_limit) +# # cpu_limit = "0-%d" % (multiprocessing.cpu_count() - 1) +# # logger.info('Running on CPUs:', cpu_limit) +# for dir_item in ['logs', 'db']: +# try: +# os.mkdir(os.path.abspath(dir_item)) +# except Exception as e: +# pass + +# if test_type == "local": +# volumes = { +# os.path.abspath('conf'): +# {'bind': '/opt/milvus/conf', 'mode': 'ro'}, +# os.path.abspath('logs'): +# {'bind': '/opt/milvus/logs', 'mode': 'rw'}, +# os.path.abspath('db'): +# {'bind': '/opt/milvus/db', 'mode': 'rw'}, +# } +# elif test_type == "remote": +# if volume_name is None: +# raise Exception("No volume name") +# remote_log_dir = volume_name+'/logs' +# remote_db_dir = volume_name+'/db' + +# for dir_item in [remote_log_dir, remote_db_dir]: +# if not os.path.isdir(dir_item): +# os.makedirs(dir_item, exist_ok=True) +# volumes = { +# os.path.abspath('conf'): +# {'bind': '/opt/milvus/conf', 'mode': 'ro'}, +# remote_log_dir: +# {'bind': '/opt/milvus/logs', 'mode': 'rw'}, +# remote_db_dir: +# {'bind': '/opt/milvus/db', 'mode': 'rw'} +# } +# # add volumes +# if db_slave and isinstance(db_slave, int): +# for i in range(2, db_slave+1): +# remote_db_dir = volume_name+'/data'+str(i) +# if not os.path.isdir(remote_db_dir): +# os.makedirs(remote_db_dir, exist_ok=True) +# volumes[remote_db_dir] = {'bind': '/opt/milvus/data'+str(i), 'mode': 'rw'} + +# container = client.containers.run( +# image, +# volumes=volumes, +# runtime="nvidia", +# ports={'19530/tcp': 19530, '8080/tcp': 8080}, +# # environment=["OMP_NUM_THREADS=48"], +# # cpuset_cpus=cpu_limit, +# # mem_limit=mem_limit, +# # environment=[""], +# detach=True) + +# def stream_logs(): +# for line in container.logs(stream=True): +# logger.info(colors.color(line.decode().rstrip(), fg='blue')) + +# if sys.version_info >= (3, 0): +# t = threading.Thread(target=stream_logs, daemon=True) +# else: +# t = threading.Thread(target=stream_logs) +# t.daemon = True +# t.start() + +# logger.info('Container: %s started' % container) +# return container +# # exit_code = container.wait(timeout=timeout) +# # # Exit if exit code +# # if exit_code == 0: +# # return container +# # elif exit_code is not None: +# # print(colors.color(container.logs().decode(), fg='red')) + +# def restart_server(container): +# client = docker.APIClient(base_url='unix://var/run/docker.sock') + +# client.restart(container.name) +# logger.info('Container: %s restarted' % container.name) +# return container + + +# def remove_container(container): +# container.remove(force=True) +# logger.info('Container: %s removed' % container) + + +# def remove_all_containers(image): +# client = docker.from_env() +# try: +# for container in client.containers.list(): +# if image in container.image.tags: +# container.stop(timeout=30) +# container.remove(force=True) +# except Exception as e: +# logger.error("Containers removed failed") + + +# def container_exists(image): +# ''' +# Check if container existed with the given image name +# @params: image name +# @return: container if exists +# ''' +# res = False +# client = docker.from_env() +# for container in client.containers.list(): +# if image in container.image.tags: +# # True +# res = container +# return res diff --git a/tests/milvus_benchmark/executors/__init__.py b/tests/milvus_benchmark/executors/__init__.py new file mode 100644 index 0000000000..dc96d174c5 --- /dev/null +++ b/tests/milvus_benchmark/executors/__init__.py @@ -0,0 +1,3 @@ + +class BaseExecutor(object): + pass \ No newline at end of file diff --git a/tests/milvus_benchmark/executors/shell.py b/tests/milvus_benchmark/executors/shell.py new file mode 100644 index 0000000000..cb425b9b54 --- /dev/null +++ b/tests/milvus_benchmark/executors/shell.py @@ -0,0 +1,4 @@ +from . import BaseExecutor + +class ShellExecutor(BaseExecutor): + pass \ No newline at end of file diff --git a/tests/milvus_benchmark/handlers/__init__.py b/tests/milvus_benchmark/handlers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/milvus_benchmark/helm_utils.py b/tests/milvus_benchmark/helm_utils.py new file mode 100644 index 0000000000..3efeee3cbc --- /dev/null +++ b/tests/milvus_benchmark/helm_utils.py @@ -0,0 +1,326 @@ +import os +import pdb +import time +import logging +from yaml import full_load, dump +import utils + +logger = logging.getLogger("milvus_benchmark.utils") +REGISTRY_URL = "registry.zilliz.com/milvus/engine" + + +def retry(times): + """ + This decorator prints the execution time for the decorated function. + """ + def wrapper(func): + def newfn(*args, **kwargs): + attempt = 0 + while attempt < times: + try: + logger.info(attempt) + return func(*args, **kwargs) + except Exception as e: + logger.info(str(e)) + time.sleep(3) + attempt += 1 + return func(*args, **kwargs) + return newfn + return wrapper + + +@utils.retry(3) +def get_host_cpus(hostname): + from kubernetes import client, config + config.load_kube_config() + client.rest.logger.setLevel(logging.WARNING) + v1 = client.CoreV1Api() + cpus = v1.read_node(hostname).status.allocatable.get("cpu") + return cpus + + +# update values.yaml +def update_values(file_path, deploy_mode, hostname, milvus_config, server_config=None): + if not os.path.isfile(file_path): + raise Exception('File: %s not found' % file_path) + # 　bak values.yaml + file_name = os.path.basename(file_path) + bak_file_name = file_name + ".bak" + file_parent_path = os.path.dirname(file_path) + bak_file_path = file_parent_path + '/' + bak_file_name + if os.path.exists(bak_file_path): + os.system("cp %s %s" % (bak_file_path, file_path)) + else: + os.system("cp %s %s" % (file_path, bak_file_path)) + with open(file_path) as f: + values_dict = full_load(f) + f.close() + + for k, v in milvus_config.items(): + if k.find("primary_path") != -1: + suffix_path = milvus_config["suffix_path"] if "suffix_path" in milvus_config else None + path_value = v + if suffix_path: + path_value = v + "_" + str(int(time.time())) + values_dict["primaryPath"] = path_value + values_dict['wal']['path'] = path_value + "/wal" + values_dict['logs']['path'] = path_value + "/logs" + # elif k.find("use_blas_threshold") != -1: + # values_dict['useBLASThreshold'] = int(v) + elif k.find("gpu_search_threshold") != -1: + values_dict['gpu']['gpuSearchThreshold'] = int(v) + elif k.find("cpu_cache_capacity") != -1: + values_dict['cache']['cacheSize'] = v + # elif k.find("cache_insert_data") != -1: + # values_dict['cache']['cacheInsertData'] = v + elif k.find("insert_buffer_size") != -1: + values_dict['cache']['insertBufferSize'] = v + elif k.find("gpu_resource_config.enable") != -1: + values_dict['gpu']['enabled'] = v + elif k.find("gpu_resource_config.cache_capacity") != -1: + values_dict['gpu']['cacheSize'] = v + elif k.find("build_index_resources") != -1: + values_dict['gpu']['buildIndexDevices'] = v + elif k.find("search_resources") != -1: + values_dict['gpu']['searchDevices'] = v + # wal + elif k.find("auto_flush_interval") != -1: + values_dict['storage']['autoFlushInterval'] = v + elif k.find("wal_enable") != -1: + values_dict['wal']['enabled'] = v + + # if values_dict['nodeSelector']: + # logger.warning("nodeSelector has been set: %s" % str(values_dict['engine']['nodeSelector'])) + # return + values_dict["wal"]["recoveryErrorIgnore"] = True + # enable monitor + values_dict["metrics"]["enabled"] = True + values_dict["metrics"]["address"] = "192.168.1.237" + values_dict["metrics"]["port"] = 9091 + # Using sqlite for single mode + if deploy_mode == "single": + values_dict["mysql"]["enabled"] = False + + # update values.yaml with the given host + if hostname: + values_dict['nodeSelector'] = {'kubernetes.io/hostname': hostname} + cpus = server_config["cpus"] + + # set limit/request cpus in resources + values_dict["image"]['resources'] = { + "limits": { + # "cpu": str(int(cpus)) + ".0" + "cpu": str(int(cpus)) + ".0" + }, + "requests": { + "cpu": str(int(cpus) - 1) + ".0" + } + } + values_dict['tolerations'] = [{ + "key": "worker", + "operator": "Equal", + "value": "performance", + "effect": "NoSchedule" + }] + # add extra volumes + values_dict['extraVolumes'] = [{ + 'name': 'test', + 'flexVolume': { + 'driver': "fstab/cifs", + 'fsType': "cifs", + 'secretRef': { + 'name': "cifs-test-secret" + }, + 'options': { + 'networkPath': "//192.168.1.126/test", + 'mountOptions': "vers=1.0" + } + } + }] + values_dict['extraVolumeMounts'] = [{ + 'name': 'test', + 'mountPath': '/test' + }] + + # add extra volumes for mysql + # values_dict['mysql']['persistence']['enabled'] = True + # values_dict['mysql']['configurationFilesPath'] = "/etc/mysql/mysql.conf.d/" + # values_dict['mysql']['imageTag'] = '5.6' + # values_dict['mysql']['securityContext'] = { + # 'enabled': True} + # mysql_db_path = "/test" + if deploy_mode == "shards": + # mount_path = values_dict["primaryPath"]+'/data' + # long_str = '- name: test-mysql\n flexVolume:\n driver: fstab/cifs\n fsType: cifs\n secretRef:\n name: cifs-test-secret\n options:\n networkPath: //192.168.1.126/test\n mountOptions: vers=1.0' + # values_dict['mysql']['extraVolumes'] = literal_str(long_str) + # long_str_2 = "- name: test-mysql\n mountPath: %s" % mysql_db_path + # values_dict['mysql']['extraVolumeMounts'] = literal_str(long_str_2) + # mysql_cnf_str = '[mysqld]\npid-file=%s/mysql.pid\ndatadir=%s' % (mount_path, mount_path) + # values_dict['mysql']['configurationFiles'] = {} + # values_dict['mysql']['configurationFiles']['mysqld.cnf'] = literal_str(mysql_cnf_str) + + values_dict['mysql']['enabled'] = False + values_dict['externalMysql']['enabled'] = True + values_dict['externalMysql']["ip"] = "192.168.1.197" + values_dict['externalMysql']["port"] = 3306 + values_dict['externalMysql']["user"] = "root" + values_dict['externalMysql']["password"] = "Fantast1c" + values_dict['externalMysql']["database"] = "db" + + # logger.debug(values_dict) + # print(dump(values_dict)) + with open(file_path, 'w') as f: + dump(values_dict, f, default_flow_style=False) + f.close() + # DEBUG + with open(file_path) as f: + for line in f.readlines(): + line = line.strip("\n") + + +# deploy server +def helm_install_server(helm_path, deploy_mode, image_tag, image_type, name, namespace): + timeout = 300 + logger.debug("Server deploy mode: %s" % deploy_mode) + host = "%s.%s.svc.cluster.local" % (name, namespace) + if deploy_mode == "single": + install_cmd = "helm install --wait --timeout %ds \ + --set image.repository=%s \ + --set image.tag=%s \ + --set image.pullPolicy=Always \ + --set service.type=ClusterIP \ + -f ci/filebeat/values.yaml \ + --namespace %s \ + %s ." % (timeout, REGISTRY_URL, image_tag, namespace, name) + elif deploy_mode == "shards": + install_cmd = "helm install --wait --timeout %ds \ + --set cluster.enabled=true \ + --set persistence.enabled=true \ + --set mishards.image.tag=test \ + --set mishards.image.pullPolicy=Always \ + --set image.repository=%s \ + --set image.tag=%s \ + --set image.pullPolicy=Always \ + --set service.type=ClusterIP \ + -f ci/filebeat/values.yaml \ + --namespace %s \ + %s ." % (timeout, REGISTRY_URL, image_tag, namespace, name) + logger.debug(install_cmd) + logger.debug(host) + if os.system("cd %s && %s" % (helm_path, install_cmd)): + logger.error("Helm install failed: %s" % name) + return None + time.sleep(5) + # config.load_kube_config() + # v1 = client.CoreV1Api() + # pod_name = None + # pod_id = None + # pods = v1.list_namespaced_pod(namespace) + # for i in pods.items: + # if i.metadata.name.find(name) != -1: + # pod_name = i.metadata.name + # pod_ip = i.status.pod_ip + # logger.debug(pod_name) + # logger.debug(pod_ip) + # return pod_name, pod_ip + return host + + +# delete server +def helm_del_server(name, namespace): + # logger.debug("Sleep 600s before uninstall server") + # time.sleep(600) + del_cmd = "helm uninstall -n milvus %s" % name + logger.debug(del_cmd) + if os.system(del_cmd): + logger.error("Helm delete name:%s failed" % name) + return False + return True + + +def restart_server(helm_release_name, namespace): + res = True + timeout = 120000 + # service_name = "%s.%s.svc.cluster.local" % (helm_release_name, namespace) + config.load_kube_config() + v1 = client.CoreV1Api() + pod_name = None + # config_map_names = v1.list_namespaced_config_map(namespace, pretty='true') + # body = {"replicas": 0} + pods = v1.list_namespaced_pod(namespace) + for i in pods.items: + if i.metadata.name.find(helm_release_name) != -1 and i.metadata.name.find("mysql") == -1: + pod_name = i.metadata.name + break + # v1.patch_namespaced_config_map(config_map_name, namespace, body, pretty='true') + # status_res = v1.read_namespaced_service_status(helm_release_name, namespace, pretty='true') + logger.debug("Pod name: %s" % pod_name) + if pod_name is not None: + try: + v1.delete_namespaced_pod(pod_name, namespace) + except Exception as e: + logging.error(str(e)) + logging.error("Exception when calling CoreV1Api->delete_namespaced_pod") + res = False + return res + logging.error("Sleep 10s after pod deleted") + time.sleep(10) + # check if restart successfully + pods = v1.list_namespaced_pod(namespace) + for i in pods.items: + pod_name_tmp = i.metadata.name + logging.error(pod_name_tmp) + if pod_name_tmp == pod_name: + continue + elif pod_name_tmp.find(helm_release_name) == -1 or pod_name_tmp.find("mysql") != -1: + continue + else: + status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true') + logging.error(status_res.status.phase) + start_time = time.time() + ready_break = False + while time.time() - start_time <= timeout: + logging.error(time.time()) + status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true') + if status_res.status.phase == "Running": + logging.error("Already running") + ready_break = True + break + else: + time.sleep(5) + if time.time() - start_time > timeout: + logging.error("Restart pod: %s timeout" % pod_name_tmp) + res = False + return res + if ready_break: + break + else: + raise Exception("Pod: %s not found" % pod_name) + follow = True + pretty = True + previous = True # bool | Return previous terminated container logs. Defaults to false. (optional) + since_seconds = 56 # int | A relative time in seconds before the current time from which to show logs. If this value precedes the time a pod was started, only logs since the pod start will be returned. If this value is in the future, no logs will be returned. Only one of sinceSeconds or sinceTime may be specified. (optional) + timestamps = True # bool | If true, add an RFC3339 or RFC3339Nano timestamp at the beginning of every line of log output. Defaults to false. (optional) + container = "milvus" + # start_time = time.time() + # while time.time() - start_time <= timeout: + # try: + # api_response = v1.read_namespaced_pod_log(pod_name_tmp, namespace, container=container, follow=follow, + # pretty=pretty, previous=previous, since_seconds=since_seconds, + # timestamps=timestamps) + # logging.error(api_response) + # return res + # except Exception as e: + # logging.error("Exception when calling CoreV1Api->read_namespaced_pod_log: %s\n" % e) + # # waiting for server start + # time.sleep(2) + # # res = False + # # return res + # if time.time() - start_time > timeout: + # logging.error("Restart pod: %s timeout" % pod_name_tmp) + # res = False + return res + + +if __name__ == '__main__': + update_values("", "shards", None, None) diff --git a/tests/milvus_benchmark/k8s_runner.py b/tests/milvus_benchmark/k8s_runner.py index 1afc04c9f7..1935d821fe 100644 --- a/tests/milvus_benchmark/k8s_runner.py +++ b/tests/milvus_benchmark/k8s_runner.py @@ -6,21 +6,32 @@ import re import random import traceback import json +import csv +import threading from multiprocessing import Process import numpy as np +from milvus import DataType from yaml import full_load, dump +import concurrent.futures + +import locust_user from client import MilvusClient import parser from runner import Runner from milvus_metrics.api import report from milvus_metrics.models import Env, Hardware, Server, Metric +import helm_utils import utils + logger = logging.getLogger("milvus_benchmark.k8s_runner") namespace = "milvus" +default_port = 19530 DELETE_INTERVAL_TIME = 5 # INSERT_INTERVAL = 100000 INSERT_INTERVAL = 50000 +BIG_FLUSH_INTERVAL = 3600 +DEFAULT_FLUSH_INTERVAL = 1 timestamp = int(time.time()) default_path = "/var/lib/milvus" @@ -29,30 +40,46 @@ class K8sRunner(Runner): """run docker mode""" def __init__(self): super(K8sRunner, self).__init__() - self.name = utils.get_unique_name() + self.service_name = utils.get_unique_name() self.host = None - self.ip = None + self.port = default_port self.hostname = None self.env_value = None - - def init_env(self, server_config, server_host, image_type, image_tag): + self.hardware = None + + def init_env(self, milvus_config, server_config, server_host, deploy_mode, image_type, image_tag): + logger.debug("Tests run on server host:") + logger.debug(server_host) self.hostname = server_host + if self.hostname: + cpus = helm_utils.get_host_cpus(self.hostname) + logger.debug(type(cpus)) + if server_config: + if "cpus" in server_config.keys(): + cpus = min(server_config["cpus"], int(cpus)) + else: + server_config.update({"cpus": cpus}) + else: + server_config = {"cpus": cpus} + self.hardware = Hardware(name=self.hostname, cpus=cpus) # update values - helm_path = os.path.join(os.getcwd(), "../milvus-helm") + helm_path = os.path.join(os.getcwd(), "../milvus-helm/charts/milvus") values_file_path = helm_path+"/values.yaml" if not os.path.exists(values_file_path): raise Exception("File %s not existed" % values_file_path) - utils.update_values(values_file_path, server_host, server_config) + if milvus_config: + helm_utils.update_values(values_file_path, deploy_mode, server_host, milvus_config, server_config) try: logger.debug("Start install server") - self.host, self.ip = utils.helm_install_server(helm_path, image_tag, image_type, self.name, namespace) + self.host = helm_utils.helm_install_server(helm_path, deploy_mode, image_tag, image_type, self.service_name, namespace) except Exception as e: - logger.error("Helm install server failed: %s" % str(e)) + logger.error("Helm install server failed: %s" % (str(e))) logger.error(traceback.format_exc()) + logger.debug(milvus_config) self.clean_up() return False + logger.debug(server_config) # for debugging - # self.host = "192.168.1.101" if not self.host: logger.error("Helm install server failed") self.clean_up() @@ -60,15 +87,15 @@ class K8sRunner(Runner): return True def clean_up(self): - logger.debug(self.name) - utils.helm_del_server(self.name, namespace) + logger.debug("Start clean up: %s" % self.service_name) + helm_utils.helm_del_server(self.service_name, namespace) - def report_wrapper(self, milvus_instance, env_value, hostname, collection_info, index_info, search_params): + def report_wrapper(self, milvus_instance, env_value, hostname, collection_info, index_info, search_params, run_params=None, server_config=None): metric = Metric() metric.set_run_id(timestamp) metric.env = Env(env_value) metric.env.OMP_NUM_THREADS = 0 - metric.hardware = Hardware(name=hostname) + metric.hardware = self.hardware server_version = milvus_instance.get_server_version() server_mode = milvus_instance.get_server_mode() commit = milvus_instance.get_server_commit() @@ -76,28 +103,34 @@ class K8sRunner(Runner): metric.collection = collection_info metric.index = index_info metric.search = search_params + metric.run_params = run_params return metric def run(self, run_type, collection): logger.debug(run_type) logger.debug(collection) - collection_name = collection["collection_name"] - milvus_instance = MilvusClient(collection_name=collection_name, ip=self.ip) - self.env_value = milvus_instance.get_server_config() + collection_name = collection["collection_name"] if "collection_name" in collection else None + milvus_instance = MilvusClient(collection_name=collection_name, host=self.host) + # TODO: enable + self.env_value = milvus_instance.get_server_config() # ugly implemention + self.env_value = utils.convert_nested(self.env_value) self.env_value.pop("logs") + self.env_value.pop("network") if run_type == "insert_performance": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) ni_per = collection["ni_per"] build_index = collection["build_index"] if milvus_instance.exists_collection(): - milvus_instance.delete() + milvus_instance.drop() time.sleep(10) index_info = {} search_params = {} - milvus_instance.create_collection(collection_name, dimension, index_file_size, metric_type) + vector_type = self.get_vector_type(data_type) + other_fields = collection["other_fields"] if "other_fields" in collection else None + milvus_instance.create_collection(dimension, segment_row_limit, data_type=vector_type, other_fields=other_fields) if build_index is True: index_type = collection["index_type"] index_param = collection["index_param"] @@ -105,15 +138,25 @@ class K8sRunner(Runner): "index_type": index_type, "index_param": index_param } - milvus_instance.create_index(index_type, index_param) + index_field_name = utils.get_default_field_name(vector_type) + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) logger.debug(milvus_instance.describe_index()) res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per) - logger.info(res) - milvus_instance.flush() + flush_time = 0.0 + if "flush" in collection and collection["flush"] == "no": + logger.debug("No manual flush") + else: + start_time = time.time() + milvus_instance.flush() + flush_time = time.time() - start_time + logger.debug(milvus_instance.count()) collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": collection_name + "dataset_name": collection_name, + "other_fields": other_fields, + "segment_row_limit": segment_row_limit, + "ni_per": ni_per } metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_params) metric.metrics = { @@ -121,53 +164,24 @@ class K8sRunner(Runner): "value": { "total_time": res["total_time"], "qps": res["qps"], - "ni_time": res["ni_time"] + "ni_time": res["ni_time"], + "flush_time": flush_time } } report(metric) if build_index is True: logger.debug("Start build index for last file") - milvus_instance.create_index(index_type, index_param) - logger.debug(milvus_instance.describe_index()) - - if run_type == "insert_flush_performance": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) - ni_per = collection["ni_per"] - if milvus_instance.exists_collection(): - milvus_instance.delete() - time.sleep(10) - index_info = {} - search_params = {} - milvus_instance.create_collection(collection_name, dimension, index_file_size, metric_type) - res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per) - logger.info(res) - logger.debug(milvus_instance.count()) - start_time = time.time() - milvus_instance.flush() - end_time = time.time() - logger.debug(milvus_instance.count()) - collection_info = { - "dimension": dimension, - "metric_type": metric_type, - "dataset_name": collection_name - } - metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_params) - metric.metrics = { - "type": run_type, - "value": { - "flush_time": round(end_time - start_time, 1) - } - } - report(metric) + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) elif run_type == "build_performance": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) index_type = collection["index_type"] index_param = collection["index_param"] collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": collection_name + "dataset_name": collection_name, + "segment_row_limit": segment_row_limit } index_info = { "index_type": index_type, @@ -177,12 +191,14 @@ class K8sRunner(Runner): logger.error("Table name: %s not existed" % collection_name) return search_params = {} + vector_type = self.get_vector_type(data_type) + index_field_name = utils.get_default_field_name(vector_type) start_time = time.time() # drop index logger.debug("Drop index") - milvus_instance.drop_index() + milvus_instance.drop_index(index_field_name) start_mem_usage = milvus_instance.get_mem_info()["memory_used"] - milvus_instance.create_index(index_type, index_param) + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) logger.debug(milvus_instance.describe_index()) logger.debug(milvus_instance.count()) end_time = time.time() @@ -200,15 +216,18 @@ class K8sRunner(Runner): report(metric) elif run_type == "delete_performance": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) ni_per = collection["ni_per"] + auto_flush = collection["auto_flush"] if "auto_flush" in collection else True search_params = {} collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": collection_name + "dataset_name": collection_name, + "segment_row_limit": segment_row_limit } if not milvus_instance.exists_collection(): + logger.error(milvus_instance.show_collections()) logger.error("Table name: %s not existed" % collection_name) return length = milvus_instance.count() @@ -220,79 +239,274 @@ class K8sRunner(Runner): milvus_instance.preload_collection() start_mem_usage = milvus_instance.get_mem_info()["memory_used"] start_time = time.time() + if auto_flush is False: + milvus_instance.set_config("storage", "auto_flush_interval", BIG_FLUSH_INTERVAL) for i in range(loops): - delete_ids = ids[i*ni_per : i*ni_per+ni_per] + delete_ids = ids[i*ni_per: i*ni_per+ni_per] logger.debug("Delete %d - %d" % (delete_ids[0], delete_ids[-1])) - milvus_instance.delete_vectors(delete_ids) + milvus_instance.delete(delete_ids) + logger.debug("Table row counts: %d" % milvus_instance.count()) + logger.debug("Table row counts: %d" % milvus_instance.count()) + start_flush_time = time.time() + milvus_instance.flush() + end_flush_time = time.time() + end_time = time.time() + end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.debug("Table row counts: %d" % milvus_instance.count()) + milvus_instance.set_config("storage", "auto_flush_interval", DEFAULT_FLUSH_INTERVAL) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_params) + delete_time = round(end_time - start_time, 1) + metric.metrics = { + "type": "delete_performance", + "value": { + "delete_time": delete_time, + "start_mem_usage": start_mem_usage, + "end_mem_usage": end_mem_usage, + "diff_mem": end_mem_usage - start_mem_usage, + "qps": round(collection_size / delete_time, 1) + } + } + if auto_flush is False: + flush_time = round(end_flush_time - start_flush_time, 1) + metric.metrics["value"].update({"flush_time": flush_time}) + report(metric) + + elif run_type == "compact_performance": + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) + search_params = {} + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name, + "segment_row_limit": segment_row_limit + } + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + length = milvus_instance.count() + logger.info(length) + index_info = milvus_instance.describe_index() + logger.info(index_info) + ids = [i for i in range(length)] + loops = int(length / segment_row_limit) + delete_ni_per = segment_row_limit // 2 + ni_per = segment_row_limit + milvus_instance.preload_collection() + for i in range(loops): + delete_ids = ids[i * ni_per: i * ni_per + delete_ni_per] + logger.debug("Delete %d - %d" % (delete_ids[0], delete_ids[-1])) + milvus_instance.delete(delete_ids) milvus_instance.flush() logger.debug("Table row counts: %d" % milvus_instance.count()) logger.debug("Table row counts: %d" % milvus_instance.count()) milvus_instance.flush() + start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + start_time = time.time() + milvus_instance.compact() end_time = time.time() end_mem_usage = milvus_instance.get_mem_info()["memory_used"] logger.debug("Table row counts: %d" % milvus_instance.count()) metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_params) + compact_time = round(end_time - start_time, 1) metric.metrics = { - "type": "delete_performance", + "type": "compact_performance", "value": { - "delete_time": round(end_time - start_time, 1), + "compact_time": compact_time, "start_mem_usage": start_mem_usage, "end_mem_usage": end_mem_usage, - "diff_mem": end_mem_usage - start_mem_usage + "diff_mem": end_mem_usage - start_mem_usage, } } report(metric) - elif run_type == "search_performance": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) - run_count = collection["run_count"] - top_ks = collection["top_ks"] - nqs = collection["nqs"] - search_params = collection["search_params"] + elif run_type == "get_ids_performance": + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) + ids_length_per_segment = collection["ids_length_per_segment"] + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": collection_name + "dataset_name": collection_name, + "segment_row_limit": segment_row_limit + } + search_params = {} + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + for ids_num in ids_length_per_segment: + segment_num, get_ids = milvus_instance.get_rand_ids_each_segment(ids_num) + start_time = time.time() + get_res = milvus_instance.get_entities(get_ids) + total_time = time.time() - start_time + avg_time = total_time / segment_num + run_params = {"ids_num": ids_num} + logger.info("Segment num: %d, ids num per segment: %d, run_time: %f" % (segment_num, ids_num, total_time)) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_params, run_params=run_params) + metric.metrics = { + "type": run_type, + "value": { + "total_time": round(total_time, 1), + "avg_time": round(avg_time, 1) + } + } + report(metric) + + elif run_type == "search_performance": + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) + run_count = collection["run_count"] + top_ks = collection["top_ks"] + nqs = collection["nqs"] + # filter_query = collection["filter"] if "filter" in collection else None + filters = collection["filters"] if "filters" in collection else [] + filter_query = [] + search_params = collection["search_params"] + fields = self.get_fields(milvus_instance, collection_name) + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name, + "segment_row_limit": segment_row_limit, + "fields": fields } - # fro debugging - # time.sleep(3600) if not milvus_instance.exists_collection(): logger.error("Table name: %s not existed" % collection_name) return + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) logger.info(milvus_instance.count()) index_info = milvus_instance.describe_index() logger.info(index_info) milvus_instance.preload_collection() logger.info("Start warm up query") - res = self.do_query(milvus_instance, collection_name, [1], [1], 2, search_param=search_params[0]) + res = self.do_query(milvus_instance, collection_name, vec_field_name, [1], [1], 2, search_param=search_params[0], filter_query=filter_query) logger.info("End warm up query") for search_param in search_params: logger.info("Search param: %s" % json.dumps(search_param)) - res = self.do_query(milvus_instance, collection_name, top_ks, nqs, run_count, search_param) - headers = ["Nq/Top-k"] - headers.extend([str(top_k) for top_k in top_ks]) - logger.info("Search param: %s" % json.dumps(search_param)) - utils.print_table(headers, nqs, res) - for index_nq, nq in enumerate(nqs): - for index_top_k, top_k in enumerate(top_ks): - search_param_group = { - "nq": nq, - "topk": top_k, - "search_param": search_param - } - search_time = res[index_nq][index_top_k] - metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_param_group) - metric.metrics = { - "type": "search_performance", - "value": { - "search_time": search_time - } - } - report(metric) + if not filters: + filters.append(None) + for filter in filters: + filter_param = [] + if isinstance(filter, dict) and "range" in filter: + filter_query.append(eval(filter["range"])) + filter_param.append(filter["range"]) + if isinstance(filter, dict) and "term" in filter: + filter_query.append(eval(filter["term"])) + filter_param.append(filter["term"]) + logger.info("filter param: %s" % json.dumps(filter_param)) + res = self.do_query(milvus_instance, collection_name, vec_field_name, top_ks, nqs, run_count, search_param, filter_query=filter_query) + headers = ["Nq/Top-k"] + headers.extend([str(top_k) for top_k in top_ks]) + logger.info("Search param: %s" % json.dumps(search_param)) + utils.print_table(headers, nqs, res) + for index_nq, nq in enumerate(nqs): + for index_top_k, top_k in enumerate(top_ks): + search_param_group = { + "nq": nq, + "topk": top_k, + "search_param": search_param, + "filter": filter_param + } + search_time = res[index_nq][index_top_k] + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_param_group) + metric.metrics = { + "type": "search_performance", + "value": { + "search_time": search_time + } + } + report(metric) + + elif run_type == "locust_insert_stress": + pass + + elif run_type == "locust_search_performance": + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) + ni_per = collection["ni_per"] + build_index = collection["build_index"] + if milvus_instance.exists_collection(): + milvus_instance.drop() + time.sleep(10) + index_info = {} + search_params = {} + vector_type = self.get_vector_type(data_type) + index_field_name = utils.get_default_field_name(vector_type) + milvus_instance.create_collection(dimension, segment_row_limit, data_type=vector_type, other_fields=None) + if build_index is True: + index_type = collection["index_type"] + index_param = collection["index_param"] + index_info = { + "index_type": index_type, + "index_param": index_param + } + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + logger.debug(milvus_instance.describe_index()) + res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per) + if "flush" in collection and collection["flush"] == "no": + logger.debug("No manual flush") + else: + milvus_instance.flush() + if build_index is True: + logger.debug("Start build index for last file") + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + logger.debug(milvus_instance.describe_index()) + logger.debug("Table row counts: %d" % milvus_instance.count()) + real_metric_type = utils.metric_type_trans(metric_type) + ### spawn locust requests + task = collection["task"] + connection_type = "single" + connection_num = task["connection_num"] + if connection_num > 1: + connection_type = "multi" + clients_num = task["clients_num"] + hatch_rate = task["hatch_rate"] + during_time = utils.timestr_to_int(task["during_time"]) + task_types = task["types"] + # """ + # task: + # connection_num: 1 + # clients_num: 100 + # hatch_rate: 2 + # during_time: 5m + # types: + # - + # type: query + # weight: 1 + # params: + # top_k: 10 + # nq: 1 + # # filters: + # # - + # # range: + # # int64: + # # LT: 0 + # # GT: 1000000 + # search_param: + # nprobe: 16 + # """ + run_params = {"tasks": {}, "clients_num": clients_num, "spawn_rate": hatch_rate, "during_time": during_time} + for task_type in task_types: + run_params["tasks"].update({task_type["type"]: task_type["weight"] if "weight" in task_type else 1}) + + #. collect stats + locust_stats = locust_user.locust_executor(self.host, self.port, collection_name, run_params=run_params) + logger.info(locust_stats) + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name, + "segment_row_limit": segment_row_limit + } + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_params) + metric.metrics = { + "type": run_type, + "value": locust_stats} + report(metric) elif run_type == "search_ids_stability": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) search_params = collection["search_params"] during_time = collection["during_time"] ids_length = collection["ids_length"] @@ -300,7 +514,8 @@ class K8sRunner(Runner): collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": collection_name + "dataset_name": collection_name, + "segment_row_limit": segment_row_limit } if not milvus_instance.exists_collection(): logger.error("Table name: %s not existed" % collection_name) @@ -344,7 +559,7 @@ class K8sRunner(Runner): # for sift/deep datasets # TODO: enable elif run_type == "accuracy": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) search_params = collection["search_params"] # mapping to search param list search_params = self.generate_combinations(search_params) @@ -354,7 +569,8 @@ class K8sRunner(Runner): collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": collection_name + "dataset_name": collection_name, + "segment_row_limit": segment_row_limit } if not milvus_instance.exists_collection(): logger.error("Table name: %s not existed" % collection_name) @@ -364,19 +580,27 @@ class K8sRunner(Runner): logger.info(index_info) milvus_instance.preload_collection() true_ids_all = self.get_groundtruth_ids(collection_size) + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) for search_param in search_params: - for top_k in top_ks: - for nq in nqs: - total = 0 + headers = ["Nq/Top-k"] + res = [] + for nq in nqs: + for top_k in top_ks: + tmp_res = [] search_param_group = { "nq": nq, "topk": top_k, - "search_param": search_param + "search_param": search_param, + "metric_type": metric_type } logger.info("Query params: %s" % json.dumps(search_param_group)) - result_ids, result_distances = self.do_query_ids(milvus_instance, collection_name, top_k, nq, search_param=search_param) + result_ids = self.do_query_ids(milvus_instance, collection_name, vec_field_name, top_k, nq, search_param=search_param) + mem_used = milvus_instance.get_mem_info()["memory_used"] acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids) logger.info("Query accuracy: %s" % acc_value) + tmp_res.append(acc_value) + logger.info("Memory usage: %s" % mem_used) metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_param_group) metric.metrics = { "type": "accuracy", @@ -385,11 +609,16 @@ class K8sRunner(Runner): } } report(metric) + logger.info("Memory usage: %s" % mem_used) + res.append(tmp_res) + headers.extend([str(top_k) for top_k in top_ks]) + logger.info("Search param: %s" % json.dumps(search_param)) + utils.print_table(headers, nqs, res) elif run_type == "ann_accuracy": hdf5_source_file = collection["source_file"] collection_name = collection["collection_name"] - index_file_sizes = collection["index_file_sizes"] + segment_row_limits = collection["segment_row_limits"] index_types = collection["index_types"] index_params = collection["index_params"] top_ks = collection["top_ks"] @@ -409,34 +638,49 @@ class K8sRunner(Runner): dataset = utils.get_dataset(hdf5_source_file) if milvus_instance.exists_collection(collection_name): logger.info("Re-create collection: %s" % collection_name) - milvus_instance.delete() + milvus_instance.drop() time.sleep(DELETE_INTERVAL_TIME) true_ids = np.array(dataset["neighbors"]) - for index_file_size in index_file_sizes: - milvus_instance.create_collection(collection_name, dimension, index_file_size, metric_type) - logger.info(milvus_instance.describe()) + vector_type = self.get_vector_type_from_metric(metric_type) + vec_field_name = utils.get_default_field_name(vector_type) + real_metric_type = utils.metric_type_trans(metric_type) + + for segment_row_limit in segment_row_limits: + collection_info.update({"segment_row_limit": segment_row_limit}) + # re-create collection + if milvus_instance.exists_collection(collection_name): + milvus_instance.drop() + time.sleep(DELETE_INTERVAL_TIME) + milvus_instance.create_collection(dimension, segment_row_limit, data_type=vector_type) insert_vectors = self.normalize(metric_type, np.array(dataset["train"])) + if len(insert_vectors) != dataset["train"].shape[0]: + raise Exception("Row count of insert vectors: %d is not equal to dataset size: %d" % (len(insert_vectors), dataset["train"].shape[0])) + logger.debug("The row count of entities to be inserted: %d" % len(insert_vectors)) # Insert batch once # milvus_instance.insert(insert_vectors) loops = len(insert_vectors) // INSERT_INTERVAL + 1 for i in range(loops): start = i*INSERT_INTERVAL end = min((i+1)*INSERT_INTERVAL, len(insert_vectors)) - tmp_vectors = insert_vectors[start:end] if start < end: + tmp_vectors = insert_vectors[start:end] + ids = [i for i in range(start, end)] if not isinstance(tmp_vectors, list): - milvus_instance.insert(tmp_vectors.tolist(), ids=[i for i in range(start, end)]) + entities = milvus_instance.generate_entities(tmp_vectors.tolist(), ids) + res_ids = milvus_instance.insert(entities, ids=ids) else: - milvus_instance.insert(tmp_vectors, ids=[i for i in range(start, end)]) + entities = milvus_instance.generate_entities(tmp_vectors, ids) + res_ids = milvus_instance.insert(entities, ids=ids) + assert res_ids == ids milvus_instance.flush() - logger.info("Table: %s, row count: %s" % (collection_name, milvus_instance.count())) - if milvus_instance.count() != len(insert_vectors): - logger.error("Table row count is not equal to insert vectors") - return + res_count = milvus_instance.count() + logger.info("Table: %s, row count: %d" % (collection_name, res_count)) + if res_count != len(insert_vectors): + raise Exception("Table row count is not equal to insert vectors") for index_type in index_types: for index_param in index_params: logger.debug("Building index with param: %s" % json.dumps(index_param)) - milvus_instance.create_index(index_type, index_param=index_param) + milvus_instance.create_index(vec_field_name, index_type, metric_type, index_param=index_param) logger.info(milvus_instance.describe_index()) logger.info("Start preload collection: %s" % collection_name) milvus_instance.preload_collection() @@ -448,18 +692,24 @@ class K8sRunner(Runner): for search_param in search_params: for nq in nqs: query_vectors = self.normalize(metric_type, np.array(dataset["test"][:nq])) + if not isinstance(query_vectors, list): + query_vectors = query_vectors.tolist() for top_k in top_ks: search_param_group = { "nq": len(query_vectors), "topk": top_k, - "search_param": search_param + "search_param": search_param, + "metric_type": metric_type } logger.debug(search_param_group) - if not isinstance(query_vectors, list): - result = milvus_instance.query(query_vectors.tolist(), top_k, search_param=search_param) - else: - result = milvus_instance.query(query_vectors, top_k, search_param=search_param) - result_ids = result.id_array + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors, + "metric_type": real_metric_type, + "params": search_param} + }} + result = milvus_instance.query(vector_query) + result_ids = milvus_instance.get_ids(result) acc_value = self.get_recall_value(true_ids[:nq, :top_k].tolist(), result_ids) logger.info("Query ann_accuracy: %s" % acc_value) metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_param_group) @@ -472,13 +722,14 @@ class K8sRunner(Runner): report(metric) elif run_type == "search_stability": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) search_params = collection["search_params"] during_time = collection["during_time"] collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": collection_name + "dataset_name": collection_name, + "segment_row_limit": segment_row_limit } if not milvus_instance.exists_collection(): logger.error("Table name: %s not existed" % collection_name) @@ -496,6 +747,9 @@ class K8sRunner(Runner): start_row_count = milvus_instance.count() logger.debug(milvus_instance.describe_index()) logger.info(start_row_count) + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + real_metric_type = utils.metric_type_trans(metric_type) start_time = time.time() while time.time() < start_time + during_time * 60: search_param = {} @@ -505,7 +759,13 @@ class K8sRunner(Runner): search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)] logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param))) - result = milvus_instance.query(query_vectors, top_k, search_param=search_param) + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors[:nq], + "metric_type": real_metric_type, + "params": search_param} + }} + milvus_instance.query(vector_query) end_mem_usage = milvus_instance.get_mem_info()["memory_used"] metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, {}) metric.metrics = { @@ -519,56 +779,151 @@ class K8sRunner(Runner): } report(metric) + elif run_type == "loop_stability": + # init data + milvus_instance.clean_db() + pull_interval = collection["pull_interval"] + collection_num = collection["collection_num"] + concurrent = collection["concurrent"] if "concurrent" in collection else False + concurrent_num = collection_num + dimension = collection["dimension"] if "dimension" in collection else 128 + insert_xb = collection["insert_xb"] if "insert_xb" in collection else 100000 + index_types = collection["index_types"] if "index_types" in collection else ['ivf_sq8'] + index_param = {"nlist": 256} + collection_names = [] + milvus_instances_map = {} + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + ids = [i for i in range(insert_xb)] + # initialize and prepare + for i in range(collection_num): + name = utils.get_unique_name(prefix="collection_%d_" % i) + collection_names.append(name) + metric_type = random.choice(["l2", "ip"]) + segment_row_limit = random.randint(50000, 100000) + # default float_vector + milvus_instance = MilvusClient(collection_name=name, host=self.host) + milvus_instance.create_collection(dimension, segment_row_limit, other_fields=None) + index_type = random.choice(index_types) + field_name = utils.get_default_field_name() + milvus_instance.create_index(field_name, index_type, metric_type, index_param=index_param) + logger.info(milvus_instance.describe_index()) + insert_vectors = utils.normalize(metric_type, insert_vectors) + entities = milvus_instance.generate_entities(insert_vectors, ids) + res_ids = milvus_instance.insert(entities, ids=ids) + milvus_instance.flush() + milvus_instances_map.update({name: milvus_instance}) + logger.info(milvus_instance.describe_index()) + + # loop time unit: min -> s + pull_interval_seconds = pull_interval * 60 + tasks = ["insert_rand", "delete_rand", "query_rand", "flush", "compact"] + i = 1 + while True: + logger.info("Loop time: %d" % i) + start_time = time.time() + while time.time() - start_time < pull_interval_seconds: + if concurrent: + threads = [] + for name in collection_names: + task_name = random.choice(tasks) + task_run = getattr(milvus_instances_map[name], task_name) + t = threading.Thread(target=task_run, args=()) + threads.append(t) + t.start() + for t in threads: + t.join() + # with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor: + # future_results = {executor.submit(getattr(milvus_instances_map[mp[j][0]], mp[j][1])): j for j in range(concurrent_num)} + # for future in concurrent.futures.as_completed(future_results): + # future.result() + else: + tmp_collection_name = random.choice(collection_names) + task_name = random.choice(tasks) + logger.info(tmp_collection_name) + logger.info(task_name) + task_run = getattr(milvus_instances_map[tmp_collection_name], task_name) + task_run() + + logger.debug("Restart server") + helm_utils.restart_server(self.service_name, namespace) + # new connection + # for name in collection_names: + # milvus_instance = MilvusClient(collection_name=name, host=self.host) + # milvus_instances_map.update({name: milvus_instance}) + time.sleep(30) + i = i + 1 + elif run_type == "stability": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) - search_params = collection["search_params"] - insert_xb = collection["insert_xb"] - insert_interval = collection["insert_interval"] - delete_xb = collection["delete_xb"] + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) during_time = collection["during_time"] + operations = collection["operations"] collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": collection_name + "dataset_name": collection_name, + "segment_row_limit": segment_row_limit } if not milvus_instance.exists_collection(): - logger.error("Table name: %s not existed" % collection_name) - return + logger.error(milvus_instance.show_collections()) + raise Exception("Table name: %s not existed" % collection_name) logger.info(milvus_instance.count()) index_info = milvus_instance.describe_index() logger.info(index_info) - g_top_k = int(collection["top_ks"].split("-")[1]) - g_nq = int(collection["nqs"].split("-")[1]) - l_top_k = int(collection["top_ks"].split("-")[0]) - l_nq = int(collection["nqs"].split("-")[0]) - milvus_instance.preload_collection() start_mem_usage = milvus_instance.get_mem_info()["memory_used"] start_row_count = milvus_instance.count() - logger.debug(milvus_instance.describe_index()) logger.info(start_row_count) - start_time = time.time() - i = 0 - ids = [] - insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + real_metric_type = utils.metric_type_trans(metric_type) query_vectors = [[random.random() for _ in range(dimension)] for _ in range(10000)] + if "insert" in operations: + insert_xb = operations["insert"]["xb"] + if "delete" in operations: + delete_xb = operations["delete"]["xb"] + if "query" in operations: + g_top_k = int(operations["query"]["top_ks"].split("-")[1]) + l_top_k = int(operations["query"]["top_ks"].split("-")[0]) + g_nq = int(operations["query"]["nqs"].split("-")[1]) + l_nq = int(operations["query"]["nqs"].split("-")[0]) + search_params = operations["query"]["search_params"] + i = 0 + start_time = time.time() while time.time() < start_time + during_time * 60: i = i + 1 - for j in range(insert_interval): - top_k = random.randint(l_top_k, g_top_k) - nq = random.randint(l_nq, g_nq) - search_param = {} - for k, v in search_params.items(): - search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) - logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param))) - result = milvus_instance.query(query_vectors[0:nq], top_k, search_param=search_param) - count = milvus_instance.count() - insert_ids = [(count+x) for x in range(len(insert_vectors))] - ids.extend(insert_ids) - status, res = milvus_instance.insert(insert_vectors, ids=insert_ids) - logger.debug("%d, row_count: %d" % (i, milvus_instance.count())) - milvus_instance.delete_vectors(ids[-delete_xb:]) - milvus_instance.flush() - milvus_instance.compact() + q = self.gen_executors(operations) + for name in q: + try: + if name == "insert": + insert_ids = random.sample(list(range(collection_size)), insert_xb) + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + entities = milvus_instance.generate_entities(insert_vectors, insert_ids) + milvus_instance.insert(entities, ids=insert_ids) + elif name == "delete": + delete_ids = random.sample(list(range(collection_size)), delete_xb) + milvus_instance.delete(delete_ids) + elif name == "query": + top_k = random.randint(l_top_k, g_top_k) + nq = random.randint(l_nq, g_nq) + search_param = {} + for k, v in search_params.items(): + search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) + logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param))) + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors[:nq], + "metric_type": real_metric_type, + "params": search_param} + }} + result = milvus_instance.query(vector_query) + elif name in ["flush", "compact"]: + func = getattr(milvus_instance, name) + func() + logger.debug(milvus_instance.count()) + except Exception as e: + logger.error(name) + logger.error(str(e)) + raise + logger.debug("Loop time: %d" % i) end_mem_usage = milvus_instance.get_mem_info()["memory_used"] end_row_count = milvus_instance.count() metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, {}) @@ -584,7 +939,209 @@ class K8sRunner(Runner): } report(metric) + elif run_type == "locust_mix_performance": + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser( + collection_name) + ni_per = collection["ni_per"] + build_index = collection["build_index"] + vector_type = self.get_vector_type(data_type) + # drop exists collection + if milvus_instance.exists_collection(): + milvus_instance.drop() + time.sleep(10) + index_info = {} + search_params = {} + # create collection + other_fields = collection["other_fields"] if "other_fields" in collection else None + milvus_instance.create_collection(dimension, segment_row_limit, data_type=DataType.FLOAT_VECTOR, + collection_name=collection_name, other_fields=other_fields) + logger.info(milvus_instance.get_info()) + # insert params + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(ni_per)] + insert_ids = random.sample(list(range(collection_size)), ni_per) + insert_vectors = utils.normalize(metric_type, insert_vectors) + entities = milvus_instance.generate_entities(insert_vectors, insert_ids, collection_name) + milvus_instance.insert(entities, ids=insert_ids) + # index param + index_field_name = utils.get_default_field_name(vector_type) + if build_index is True: + index_type = collection["index_type"] + index_param = collection["index_param"] + index_info = { + "index_type": index_type, + "index_param": index_param + } + # insert entities + milvus_instance.insert(entities, ids=insert_ids) + logger.info(milvus_instance.get_stats()) + # flush + if "flush" in collection and collection["flush"] == "no": + logger.debug("No manual flush") + else: + milvus_instance.flush() + # create index + if build_index is True: + logger.debug("Start build index for last file") + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param) + logger.debug(milvus_instance.describe_index()) + ### spawn locust requests + task = collection["tasks"] + # generate task code + task_file = utils.get_unique_name() + task_file_script = task_file + '.py' + task_file_csv = task_file + '_stats.csv' + task_types = task["types"] + connection_type = "single" + connection_num = task["connection_num"] + if connection_num > 1: + connection_type = "multi" + clients_num = task["clients_num"] + hatch_rate = task["hatch_rate"] + during_time = task["during_time"] + def_strs = "" + for task_type in task_types: + type = task_type["type"] + weight = task_type["weight"] + if type == "flush": + def_str = """ + @task(%d) + def flush(self): + client = get_client(collection_name) + client.flush(collection_name=collection_name) + """ % weight + if type == "compact": + def_str = """ + @task(%d) + def compact(self): + client = get_client(collection_name) + client.compact(collection_name) + """ % weight + if type == "query": + def_str = """ + @task(%d) + def query(self): + client = get_client(collection_name) + params = %s + X = [[random.random() for i in range(dim)] for i in range(params["nq"])] + vector_query = {"vector": {"%s": { + "topk": params["top_k"], + "query": X, + "metric_type": "%s", + "params": params["search_param"]}}} + client.query(vector_query, filter_query=params["filters"], collection_name=collection_name) + """ % (weight, task_type["params"], index_field_name, utils.metric_type_trans(metric_type)) + if type == "insert": + def_str = """ + @task(%d) + def insert(self): + client = get_client(collection_name) + params = %s + insert_ids = random.sample(list(range(100000)), params["nb"]) + insert_vectors = [[random.random() for _ in range(dim)] for _ in range(params["nb"])] + insert_vectors = utils.normalize("l2", insert_vectors) + entities = generate_entities(insert_vectors, insert_ids) + client.insert(entities,ids=insert_ids, collection_name=collection_name) + """ % (weight, task_type["params"]) + if type == "delete": + def_str = """ + @task(%d) + def delete(self): + client = get_client(collection_name) + ids = [random.randint(1, 1000000) for i in range(1)] + client.delete(ids, collection_name) + """ % weight + def_strs += def_str + code_str = """ +import random +import json +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient +import utils + +host = '%s' +port = %s +collection_name = '%s' +dim = %s +connection_type = '%s' +m = MilvusClient(host=host, port=port) + + +def get_client(collection_name): + if connection_type == 'single': + return MilvusTask(m=m) + elif connection_type == 'multi': + return MilvusTask(connection_type='multi', host=host, port=port, collection_name=collection_name) + + +def generate_entities(vectors, ids): + return m.generate_entities(vectors, ids, collection_name) + + +class MixTask(User): + wait_time = between(0.001, 0.002) + %s + """ % (self.host, self.port, collection_name, dimension, connection_type, def_strs) + print(def_strs) + with open(task_file_script, "w+") as fd: + fd.write(code_str) + locust_cmd = "locust -f %s --headless --csv=%s -u %d -r %d -t %s --logfile /dev/null" % ( + task_file_script, + task_file, + clients_num, + hatch_rate, + during_time) + logger.info(locust_cmd) + try: + res = os.system(locust_cmd) + except Exception as e: + logger.error(str(e)) + return + # . retrieve and collect test statistics + locust_stats = None + with open(task_file_csv, newline='') as fd: + dr = csv.DictReader(fd) + for row in dr: + if row["Name"] != "Aggregated": + continue + locust_stats = row + logger.info(locust_stats) + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name, + "segment_row_limit": segment_row_limit + } + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, + search_params) + metric.metrics = { + "type": run_type, + "value": { + "during_time": during_time, + "request_count": int(locust_stats["Request Count"]), + "failure_count": int(locust_stats["Failure Count"]), + "qps": locust_stats["Requests/s"], + "min_response_time": int(locust_stats["Min Response Time"]), + "max_response_time": int(locust_stats["Max Response Time"]), + "median_response_time": int(locust_stats["Median Response Time"]), + "avg_response_time": int(locust_stats["Average Response Time"]) + } + } + report(metric) + + elif run_type == "debug": + time.sleep(7200) + default_insert_vectors = [[random.random() for _ in range(128)] for _ in range(500000)] + interval = 50000 + for loop in range(1, 7): + insert_xb = loop * interval + insert_vectors = default_insert_vectors[:insert_xb] + insert_ids = [i for i in range(insert_xb)] + entities = milvus_instance.generate_entities(insert_vectors, insert_ids) + for j in range(5): + milvus_instance.insert(entities, ids=insert_ids) + time.sleep(10) + else: - logger.warning("Run type not defined") - return - logger.debug("Test finished") + raise Exception("Run type not defined") + logger.debug("All test finished") diff --git a/tests/milvus_benchmark/local_runner.py b/tests/milvus_benchmark/local_runner.py index a79f4abfbc..a1a1a84682 100644 --- a/tests/milvus_benchmark/local_runner.py +++ b/tests/milvus_benchmark/local_runner.py @@ -1,16 +1,23 @@ import os import logging import pdb +import string import time import random import json +import csv from multiprocessing import Process import numpy as np import concurrent.futures +from queue import Queue + +import locust_user +from milvus import DataType from client import MilvusClient +from runner import Runner import utils import parser -from runner import Runner + DELETE_INTERVAL_TIME = 5 INSERT_INTERVAL = 50000 @@ -19,92 +26,135 @@ logger = logging.getLogger("milvus_benchmark.local_runner") class LocalRunner(Runner): """run local mode""" - def __init__(self, ip, port): + def __init__(self, host, port): super(LocalRunner, self).__init__() - self.ip = ip + self.host = host self.port = port def run(self, run_type, collection): logger.debug(run_type) logger.debug(collection) - collection_name = collection["collection_name"] - milvus_instance = MilvusClient(collection_name=collection_name, ip=self.ip, port=self.port) + collection_name = collection["collection_name"] if "collection_name" in collection else None + milvus_instance = MilvusClient(collection_name=collection_name, host=self.host, port=self.port) logger.info(milvus_instance.show_collections()) - env_value = milvus_instance.get_server_config() - logger.debug(env_value) + # TODO: + self.env_value = milvus_instance.get_server_config() + # ugly implemention + self.env_value = utils.convert_nested(self.env_value) + self.env_value.pop("logs") + self.env_value.pop("network") + logger.info(self.env_value) if run_type in ["insert_performance", "insert_flush_performance"]: - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) ni_per = collection["ni_per"] build_index = collection["build_index"] if milvus_instance.exists_collection(): - milvus_instance.delete() + milvus_instance.drop() time.sleep(10) - milvus_instance.create_collection(collection_name, dimension, index_file_size, metric_type) + vector_type = self.get_vector_type(data_type) + other_fields = collection["other_fields"] if "other_fields" in collection else None + milvus_instance.create_collection(dimension, segment_row_limit, data_type=vector_type, other_fields=other_fields) if build_index is True: index_type = collection["index_type"] index_param = collection["index_param"] - milvus_instance.create_index(index_type, index_param) - logger.debug(milvus_instance.describe_index()) + index_field_name = utils.get_default_field_name(vector_type) + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per) milvus_instance.flush() logger.debug("Table row counts: %d" % milvus_instance.count()) if build_index is True: logger.debug("Start build index for last file") - milvus_instance.create_index(index_type, index_param) - logger.debug(milvus_instance.describe_index()) + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) elif run_type == "delete_performance": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) ni_per = collection["ni_per"] + auto_flush = collection["auto_flush"] if "auto_flush" in collection else True if not milvus_instance.exists_collection(): logger.error(milvus_instance.show_collections()) - logger.warning("Table: %s not found" % collection_name) + logger.error("Table: %s not found" % collection_name) return length = milvus_instance.count() ids = [i for i in range(length)] loops = int(length / ni_per) + if auto_flush is False: + milvus_instance.set_config("storage", "auto_flush_interval", BIG_FLUSH_INTERVAL) for i in range(loops): - delete_ids = ids[i*ni_per : i*ni_per+ni_per] + delete_ids = ids[i*ni_per: i*ni_per+ni_per] logger.debug("Delete %d - %d" % (delete_ids[0], delete_ids[-1])) - milvus_instance.delete_vectors(delete_ids) - milvus_instance.flush() + milvus_instance.delete(delete_ids) logger.debug("Table row counts: %d" % milvus_instance.count()) logger.debug("Table row counts: %d" % milvus_instance.count()) - milvus_instance.flush() + milvus_instance.flush() + logger.debug("Table row counts: %d" % milvus_instance.count()) + + elif run_type == "compact_performance": + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) + if not milvus_instance.exists_collection(): + logger.error(milvus_instance.show_collections()) + logger.error("Table: %s not found" % collection_name) + return + length = milvus_instance.count() + ids = [i for i in range(length)] + loops = int(length / segment_row_limit) + delete_ni_per = segment_row_limit // 2 + ni_per = segment_row_limit + for i in range(loops): + delete_ids = ids[i*ni_per : i*ni_per+delete_ni_per] + logger.debug("Delete %d - %d" % (delete_ids[0], delete_ids[-1])) + milvus_instance.delete(delete_ids) + milvus_instance.flush() + logger.debug("Table row counts: %d" % milvus_instance.count()) + logger.debug("Table row counts: %d" % milvus_instance.count()) + milvus_instance.flush() + milvus_instance.compact() logger.debug("Table row counts: %d" % milvus_instance.count()) elif run_type == "build_performance": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) index_type = collection["index_type"] index_param = collection["index_param"] if not milvus_instance.exists_collection(): logger.error("Table name: %s not existed" % collection_name) return - search_params = {} - start_time = time.time() + vector_type = self.get_vector_type(data_type) + index_field_name = utils.get_default_field_name(vector_type) # drop index logger.debug("Drop index") - milvus_instance.drop_index() + milvus_instance.drop_index(index_field_name) start_mem_usage = milvus_instance.get_mem_info()["memory_used"] - milvus_instance.create_index(index_type, index_param) - logger.debug(milvus_instance.describe_index()) - logger.debug("Table row counts: %d" % milvus_instance.count()) + start_time = time.time() + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) end_time = time.time() + logger.debug("Table row counts: %d" % milvus_instance.count()) end_mem_usage = milvus_instance.get_mem_info()["memory_used"] logger.debug("Diff memory: %s, current memory usage: %s, build time: %s" % ((end_mem_usage - start_mem_usage), end_mem_usage, round(end_time - start_time, 1))) elif run_type == "search_performance": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) run_count = collection["run_count"] top_ks = collection["top_ks"] nqs = collection["nqs"] search_params = collection["search_params"] + filter_query = [] + filters = collection["filters"] if "filters" in collection else [] + # pdb.set_trace() + # ranges = collection["range"] if "range" in collection else None + # terms = collection["term"] if "term" in collection else None + # if ranges: + # filter_query.append(eval(ranges)) + # if terms: + # filter_query.append(eval(terms)) + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) # for debugging # time.sleep(3600) if not milvus_instance.exists_collection(): logger.error("Table name: %s not existed" % collection_name) return + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) logger.info(milvus_instance.count()) result = milvus_instance.describe_index() logger.info(result) @@ -113,16 +163,90 @@ class LocalRunner(Runner): logger.info(mem_usage) for search_param in search_params: logger.info("Search param: %s" % json.dumps(search_param)) - res = self.do_query(milvus_instance, collection_name, top_ks, nqs, run_count, search_param) - headers = ["Nq/Top-k"] - headers.extend([str(top_k) for top_k in top_ks]) - logger.info("Search param: %s" % json.dumps(search_param)) - utils.print_table(headers, nqs, res) - mem_usage = milvus_instance.get_mem_info()["memory_used"] - logger.info(mem_usage) + filter_param = [] + if not filters: + filters.append(None) + for filter in filters: + if isinstance(filter, dict) and "range" in filter: + filter_query.append(eval(filter["range"])) + filter_param.append(filter["range"]) + if isinstance(filter, dict) and "term" in filter: + filter_query.append(eval(filter["term"])) + filter_param.append(filter["term"]) + logger.info("filter param: %s" % json.dumps(filter_param)) + res = self.do_query(milvus_instance, collection_name, vec_field_name, top_ks, nqs, run_count, search_param, filter_query) + headers = ["Nq/Top-k"] + headers.extend([str(top_k) for top_k in top_ks]) + logger.info("Search param: %s" % json.dumps(search_param)) + utils.print_table(headers, nqs, res) + mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.info(mem_usage) + + elif run_type == "locust_search_performance": + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) + ni_per = collection["ni_per"] + build_index = collection["build_index"] + vector_type = self.get_vector_type(data_type) + index_field_name = utils.get_default_field_name(vector_type) + if build_index is True: + index_type = collection["index_type"] + index_param = collection["index_param"] + # TODO: debug + # if milvus_instance.exists_collection(): + # milvus_instance.drop() + # time.sleep(10) + # other_fields = collection["other_fields"] if "other_fields" in collection else None + # milvus_instance.create_collection(dimension, segment_row_limit, data_type=vector_type, other_fields=other_fields) + # milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + # res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per) + # milvus_instance.flush() + # logger.debug("Table row counts: %d" % milvus_instance.count()) + # if build_index is True: + # logger.debug("Start build index for last file") + # milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + real_metric_type = utils.metric_type_trans(metric_type) + ### spawn locust requests + task = collection["task"] + connection_type = "single" + connection_num = task["connection_num"] + if connection_num > 1: + connection_type = "multi" + clients_num = task["clients_num"] + hatch_rate = task["hatch_rate"] + during_time = utils.timestr_to_int(task["during_time"]) + task_types = task["types"] + # """ + # task: + # connection_num: 1 + # clients_num: 100 + # hatch_rate: 2 + # during_time: 5m + # types: + # - + # type: query + # weight: 1 + # params: + # top_k: 10 + # nq: 1 + # # filters: + # # - + # # range: + # # int64: + # # LT: 0 + # # GT: 1000000 + # search_param: + # nprobe: 16 + # """ + run_params = {"tasks": {}, "clients_num": clients_num, "spawn_rate": hatch_rate, "during_time": during_time} + for task_type in task_types: + run_params["tasks"].update({task_type["type"]: task_type["weight"] if "weight" in task_type else 1}) + + #. collect stats + locust_stats = locust_user.locust_executor(self.host, self.port, collection_name, run_params=run_params) + logger.info(locust_stats) elif run_type == "search_ids_stability": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) search_params = collection["search_params"] during_time = collection["during_time"] ids_length = collection["ids_length"] @@ -188,12 +312,12 @@ class LocalRunner(Runner): logger.info("Search param: %s" % json.dumps(search_param)) total_time = 0.0 if use_single_connection is True: - connections = [MilvusClient(collection_name=collection_name, ip=self.ip, port=self.port)] + connections = [MilvusClient(collection_name=collection_name, host=self.host, port=self.port)] with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor: future_results = {executor.submit( self.do_query_qps, connections[0], query_vectors, top_k, search_param=search_param) : index for index in range(concurrent_num)} else: - connections = [MilvusClient(collection_name=collection_name, ip=self.ip, port=self.port) for i in range(concurrent_num)] + connections = [MilvusClient(collection_name=collection_name, host=self.hos, port=self.port) for i in range(concurrent_num)] with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor: future_results = {executor.submit( self.do_query_qps, connections[index], query_vectors, top_k, search_param=search_param) : index for index in range(concurrent_num)} @@ -207,7 +331,7 @@ class LocalRunner(Runner): elif run_type == "ann_accuracy": hdf5_source_file = collection["source_file"] collection_name = collection["collection_name"] - index_file_sizes = collection["index_file_sizes"] + segment_row_limits = collection["segment_row_limits"] index_types = collection["index_types"] index_params = collection["index_params"] top_ks = collection["top_ks"] @@ -217,102 +341,179 @@ class LocalRunner(Runner): search_params = self.generate_combinations(search_params) # mapping to index param list index_params = self.generate_combinations(index_params) - data_type, dimension, metric_type = parser.parse_ann_collection_name(collection_name) dataset = utils.get_dataset(hdf5_source_file) - if milvus_instance.exists_collection(collection_name): - logger.info("Re-create collection: %s" % collection_name) - milvus_instance.delete() - time.sleep(DELETE_INTERVAL_TIME) true_ids = np.array(dataset["neighbors"]) - for index_file_size in index_file_sizes: - milvus_instance.create_collection(collection_name, dimension, index_file_size, metric_type) - logger.info(milvus_instance.describe()) + vector_type = self.get_vector_type_from_metric(metric_type) + vec_field_name = utils.get_default_field_name(vector_type) + real_metric_type = utils.metric_type_trans(metric_type) + + for segment_row_limit in segment_row_limits: + # re-create collection + if milvus_instance.exists_collection(collection_name): + milvus_instance.drop() + time.sleep(DELETE_INTERVAL_TIME) + milvus_instance.create_collection(dimension, segment_row_limit, data_type=vector_type) insert_vectors = self.normalize(metric_type, np.array(dataset["train"])) - logger.debug(len(insert_vectors)) - # Insert batch once + if len(insert_vectors) != dataset["train"].shape[0]: + raise Exception("Row count of insert vectors: %d is not equal to dataset size: %d" % (len(insert_vectors), dataset["train"].shape[0])) + logger.debug("The row count of entities to be inserted: %d" % len(insert_vectors)) + # insert batch once # milvus_instance.insert(insert_vectors) loops = len(insert_vectors) // INSERT_INTERVAL + 1 for i in range(loops): start = i*INSERT_INTERVAL end = min((i+1)*INSERT_INTERVAL, len(insert_vectors)) - tmp_vectors = insert_vectors[start:end] if start < end: + tmp_vectors = insert_vectors[start:end] + ids = [i for i in range(start, end)] if not isinstance(tmp_vectors, list): - milvus_instance.insert(tmp_vectors.tolist(), ids=[i for i in range(start, end)]) + entities = milvus_instance.generate_entities(tmp_vectors.tolist(), ids) + res_ids = milvus_instance.insert(entities, ids=ids) else: - milvus_instance.insert(tmp_vectors, ids=[i for i in range(start, end)]) - milvus_instance.flush() - logger.info("Table: %s, row count: %s" % (collection_name, milvus_instance.count())) - if milvus_instance.count() != len(insert_vectors): - logger.error("Table row count is not equal to insert vectors") - return + entities = milvus_instance.generate_entities(tmp_vectors, ids) + res_ids = milvus_instance.insert(entities, ids=ids) + assert res_ids == ids + milvus_instance.flush() + res_count = milvus_instance.count() + logger.info("Table: %s, row count: %d" % (collection_name, res_count)) + if res_count != len(insert_vectors): + raise Exception("Table row count is not equal to insert vectors") for index_type in index_types: for index_param in index_params: - logger.debug("Building index with param: %s" % json.dumps(index_param)) - milvus_instance.create_index(index_type, index_param=index_param) - logger.info(milvus_instance.describe_index()) + logger.debug("Building index with param: %s, metric_type: %s" % (json.dumps(index_param), metric_type)) + milvus_instance.create_index(vec_field_name, index_type, metric_type, index_param=index_param) logger.info("Start preload collection: %s" % collection_name) milvus_instance.preload_collection() for search_param in search_params: for nq in nqs: query_vectors = self.normalize(metric_type, np.array(dataset["test"][:nq])) + if not isinstance(query_vectors, list): + query_vectors = query_vectors.tolist() for top_k in top_ks: - logger.debug("Search nq: %d, top-k: %d, search_param: %s" % (nq, top_k, json.dumps(search_param))) - if not isinstance(query_vectors, list): - result = milvus_instance.query(query_vectors.tolist(), top_k, search_param=search_param) - else: - result = milvus_instance.query(query_vectors, top_k, search_param=search_param) - result_ids = result.id_array + logger.debug("Search nq: %d, top-k: %d, search_param: %s, metric_type: %s" % (nq, top_k, json.dumps(search_param), metric_type)) + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors, + "metric_type": real_metric_type, + "params": search_param} + }} + result = milvus_instance.query(vector_query) + result_ids = milvus_instance.get_ids(result) + # pdb.set_trace() acc_value = self.get_recall_value(true_ids[:nq, :top_k].tolist(), result_ids) logger.info("Query ann_accuracy: %s" % acc_value) - - - elif run_type == "stability": - (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + elif run_type == "accuracy": + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) search_params = collection["search_params"] - insert_xb = collection["insert_xb"] - insert_interval = collection["insert_interval"] - delete_xb = collection["delete_xb"] - # flush_interval = collection["flush_interval"] - # compact_interval = collection["compact_interval"] - during_time = collection["during_time"] + # mapping to search param list + search_params = self.generate_combinations(search_params) + + top_ks = collection["top_ks"] + nqs = collection["nqs"] + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name, + "segment_row_limit": segment_row_limit + } if not milvus_instance.exists_collection(): - logger.error(milvus_instance.show_collections()) logger.error("Table name: %s not existed" % collection_name) return - g_top_k = int(collection["top_ks"].split("-")[1]) - g_nq = int(collection["nqs"].split("-")[1]) - l_top_k = int(collection["top_ks"].split("-")[0]) - l_nq = int(collection["nqs"].split("-")[0]) + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + milvus_instance.preload_collection() + true_ids_all = self.get_groundtruth_ids(collection_size) + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + for search_param in search_params: + headers = ["Nq/Top-k"] + res = [] + for nq in nqs: + tmp_res = [] + for top_k in top_ks: + search_param_group = { + "nq": nq, + "topk": top_k, + "search_param": search_param, + "metric_type": metric_type + } + logger.info("Query params: %s" % json.dumps(search_param_group)) + result_ids = self.do_query_ids(milvus_instance, collection_name, vec_field_name, top_k, nq, search_param=search_param) + mem_used = milvus_instance.get_mem_info()["memory_used"] + acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids) + logger.info("Query accuracy: %s" % acc_value) + tmp_res.append(acc_value) + logger.info("Memory usage: %s" % mem_used) + res.append(tmp_res) + headers.extend([str(top_k) for top_k in top_ks]) + logger.info("Search param: %s" % json.dumps(search_param)) + utils.print_table(headers, nqs, res) + + elif run_type == "stability": + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name) + during_time = collection["during_time"] + operations = collection["operations"] + if not milvus_instance.exists_collection(): + logger.error(milvus_instance.show_collections()) + raise Exception("Table name: %s not existed" % collection_name) milvus_instance.preload_collection() start_mem_usage = milvus_instance.get_mem_info()["memory_used"] start_row_count = milvus_instance.count() - logger.debug(milvus_instance.describe_index()) logger.info(start_row_count) - start_time = time.time() - i = 0 - ids = [] - insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + real_metric_type = utils.metric_type_trans(metric_type) query_vectors = [[random.random() for _ in range(dimension)] for _ in range(10000)] + if "insert" in operations: + insert_xb = operations["insert"]["xb"] + if "delete" in operations: + delete_xb = operations["delete"]["xb"] + if "query" in operations: + g_top_k = int(operations["query"]["top_ks"].split("-")[1]) + l_top_k = int(operations["query"]["top_ks"].split("-")[0]) + g_nq = int(operations["query"]["nqs"].split("-")[1]) + l_nq = int(operations["query"]["nqs"].split("-")[0]) + search_params = operations["query"]["search_params"] + i = 0 + start_time = time.time() while time.time() < start_time + during_time * 60: i = i + 1 - for j in range(insert_interval): - top_k = random.randint(l_top_k, g_top_k) - nq = random.randint(l_nq, g_nq) - search_param = {} - for k, v in search_params.items(): - search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) - logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param))) - result = milvus_instance.query(query_vectors[0:nq], top_k, search_param=search_param) - count = milvus_instance.count() - insert_ids = [(count+x) for x in range(len(insert_vectors))] - ids.extend(insert_ids) - status, res = milvus_instance.insert(insert_vectors, ids=insert_ids) - logger.debug("%d, row_count: %d" % (i, milvus_instance.count())) - milvus_instance.delete_vectors(ids[-delete_xb:]) - milvus_instance.flush() - milvus_instance.compact() + q = self.gen_executors(operations) + for name in q: + try: + if name == "insert": + insert_ids = random.sample(list(range(collection_size)), insert_xb) + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + entities = milvus_instance.generate_entities(insert_vectors, insert_ids) + milvus_instance.insert(entities, ids=insert_ids) + elif name == "delete": + delete_ids = random.sample(list(range(collection_size)), delete_xb) + milvus_instance.delete(delete_ids) + elif name == "query": + top_k = random.randint(l_top_k, g_top_k) + nq = random.randint(l_nq, g_nq) + search_param = {} + for k, v in search_params.items(): + search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) + logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param))) + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors[:nq], + "metric_type": real_metric_type, + "params": search_param} + }} + result = milvus_instance.query(vector_query) + elif name in ["flush", "compact"]: + func = getattr(milvus_instance, name) + func() + logger.debug(milvus_instance.count()) + except Exception as e: + logger.error(name) + logger.error(str(e)) + raise + logger.debug("Loop time: %d" % i) end_mem_usage = milvus_instance.get_mem_info()["memory_used"] end_row_count = milvus_instance.count() metrics = { @@ -324,7 +525,233 @@ class LocalRunner(Runner): } logger.info(metrics) + elif run_type == "loop_stability": + # init data + milvus_instance.clean_db() + pull_interval = collection["pull_interval"] + collection_num = collection["collection_num"] + concurrent = collection["concurrent"] if "concurrent" in collection else False + concurrent_num = collection_num + dimension = collection["dimension"] if "dimension" in collection else 128 + insert_xb = collection["insert_xb"] if "insert_xb" in collection else 100000 + index_types = collection["index_types"] if "index_types" in collection else ['ivf_sq8'] + index_param = {"nlist": 256} + collection_names = [] + milvus_instances_map = {} + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + ids = [i for i in range(insert_xb)] + # initialize and prepare + for i in range(collection_num): + name = utils.get_unique_name(prefix="collection_%d_" % i) + collection_names.append(name) + metric_type = random.choice(["l2", "ip"]) + segment_row_limit = random.randint(50000, 100000) + # default float_vector + milvus_instance = MilvusClient(collection_name=name, host=self.host) + milvus_instance.create_collection(dimension, segment_row_limit, other_fields=None) + index_type = random.choice(index_types) + field_name = utils.get_default_field_name() + milvus_instance.create_index(field_name, index_type, metric_type, index_param=index_param) + logger.info(milvus_instance.describe_index()) + insert_vectors = utils.normalize(metric_type, insert_vectors) + entities = milvus_instance.generate_entities(insert_vectors, ids) + res_ids = milvus_instance.insert(entities, ids=ids) + milvus_instance.flush() + milvus_instances_map.update({name: milvus_instance}) + logger.info(milvus_instance.describe_index()) + + # loop time unit: min -> s + pull_interval_seconds = pull_interval * 60 + tasks = ["insert_rand", "delete_rand", "query_rand", "flush", "compact"] + i = 1 + while True: + logger.info("Loop time: %d" % i) + start_time = time.time() + while time.time() - start_time < pull_interval_seconds: + if concurrent: + threads = [] + for name in collection_names: + task_name = random.choice(tasks) + task_run = getattr(milvus_instances_map[name], task_name) + t = threading.Thread(target=task_run, args=()) + threads.append(t) + t.start() + for t in threads: + t.join() + # with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor: + # future_results = {executor.submit(getattr(milvus_instances_map[mp[j][0]], mp[j][1])): j for j in range(concurrent_num)} + # for future in concurrent.futures.as_completed(future_results): + # future.result() + else: + tmp_collection_name = random.choice(collection_names) + task_name = random.choice(tasks) + logger.info(tmp_collection_name) + logger.info(task_name) + task_run = getattr(milvus_instances_map[tmp_collection_name], task_name) + task_run() + # new connection + # for name in collection_names: + # milvus_instance = MilvusClient(collection_name=name, host=self.host) + # milvus_instances_map.update({name: milvus_instance}) + i = i + 1 + + elif run_type == "locust_mix_performance": + (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser( + collection_name) + ni_per = collection["ni_per"] + build_index = collection["build_index"] + vector_type = self.get_vector_type(data_type) + index_field_name = utils.get_default_field_name(vector_type) + # drop exists collection + if milvus_instance.exists_collection(): + milvus_instance.drop() + time.sleep(10) + # create collection + other_fields = collection["other_fields"] if "other_fields" in collection else None + milvus_instance.create_collection(dimension, segment_row_limit, data_type=DataType.FLOAT_VECTOR, collection_name=collection_name, other_fields=other_fields) + logger.info(milvus_instance.get_info()) + # insert entities + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(ni_per)] + insert_ids = random.sample(list(range(collection_size)), ni_per) + insert_vectors = utils.normalize(metric_type, insert_vectors) + entities = milvus_instance.generate_entities(insert_vectors, insert_ids, collection_name) + milvus_instance.insert(entities, ids=insert_ids) + # flush + milvus_instance.flush() + logger.info(milvus_instance.get_stats()) + logger.debug("Table row counts: %d" % milvus_instance.count()) + # create index + if build_index is True: + index_type = collection["index_type"] + index_param = collection["index_param"] + logger.debug("Start build index for last file") + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param) + logger.debug(milvus_instance.describe_index()) + # locust + task = collection["tasks"] + task_file = utils.get_unique_name() + task_file_script = task_file + '.py' + task_file_csv = task_file + '_stats.csv' + task_types = task["types"] + connection_type = "single" + connection_num = task["connection_num"] + if connection_num > 1: + connection_type = "multi" + clients_num = task["clients_num"] + hatch_rate = task["hatch_rate"] + during_time = task["during_time"] + def_strs = "" + # define def str + for task_type in task_types: + type = task_type["type"] + weight = task_type["weight"] + if type == "flush": + def_str = """ + @task(%d) + def flush(self): + client = get_client(collection_name) + client.flush(collection_name=collection_name) + """ % weight + if type == "compact": + def_str = """ + @task(%d) + def compact(self): + client = get_client(collection_name) + client.compact(collection_name) + """ % weight + if type == "query": + def_str = """ + @task(%d) + def query(self): + client = get_client(collection_name) + params = %s + X = [[random.random() for i in range(dim)] for i in range(params["nq"])] + vector_query = {"vector": {"%s": { + "topk": params["top_k"], + "query": X, + "metric_type": "%s", + "params": params["search_param"]}}} + client.query(vector_query, filter_query=params["filters"], collection_name=collection_name) + """ % (weight, task_type["params"], index_field_name, utils.metric_type_trans(metric_type)) + if type == "insert": + def_str = """ + @task(%d) + def insert(self): + client = get_client(collection_name) + params = %s + insert_ids = random.sample(list(range(100000)), params["nb"]) + insert_vectors = [[random.random() for _ in range(dim)] for _ in range(params["nb"])] + insert_vectors = utils.normalize("l2", insert_vectors) + entities = generate_entities(insert_vectors, insert_ids) + client.insert(entities,ids=insert_ids, collection_name=collection_name) + """ % (weight, task_type["params"]) + if type == "delete": + def_str = """ + @task(%d) + def delete(self): + client = get_client(collection_name) + ids = [random.randint(1, 1000000) for i in range(1)] + client.delete(ids, collection_name) + """ % weight + def_strs += def_str + print(def_strs) + # define locust code str + code_str = """ +import random +import json +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient +import utils + +host = '%s' +port = %s +collection_name = '%s' +dim = %s +connection_type = '%s' +m = MilvusClient(host=host, port=port) + + +def get_client(collection_name): + if connection_type == 'single': + return MilvusTask(m=m) + elif connection_type == 'multi': + return MilvusTask(connection_type='multi', host=host, port=port, collection_name=collection_name) + + +def generate_entities(vectors, ids): + return m.generate_entities(vectors, ids, collection_name) + + +class MixTask(User): + wait_time = between(0.001, 0.002) + %s + """ % (self.host, self.port, collection_name, dimension, connection_type, def_strs) + with open(task_file_script, "w+") as fd: + fd.write(code_str) + locust_cmd = "locust -f %s --headless --csv=%s -u %d -r %d -t %s" % ( + task_file_script, + task_file, + clients_num, + hatch_rate, + during_time) + logger.info(locust_cmd) + try: + res = os.system(locust_cmd) + except Exception as e: + logger.error(str(e)) + return + + # . retrieve and collect test statistics + metric = None + with open(task_file_csv, newline='') as fd: + dr = csv.DictReader(fd) + for row in dr: + if row["Name"] != "Aggregated": + continue + metric = row + logger.info(metric) + else: - logger.warning("Run type not defined") - return - logger.debug("Test finished") + raise Exception("Run type not defined") + logger.debug("All test finished") diff --git a/tests/milvus_benchmark/locust_file.py b/tests/milvus_benchmark/locust_file.py new file mode 100644 index 0000000000..97f1f65d42 --- /dev/null +++ b/tests/milvus_benchmark/locust_file.py @@ -0,0 +1,30 @@ + +import random +from locust import HttpUser, task, between + + +collection_name = "random_1m_2048_512_ip_sq8" +headers = {'Content-Type': "application/json"} +url = '/collections/%s/vectors' % collection_name +top_k = 2 +nq = 1 +dim = 512 +vectors = [[random.random() for _ in range(dim)] for _ in range(nq)] +data = { + "search":{ + "topk": top_k, + "vectors": vectors, + "params": { + "nprobe": 1 + } + } +} + +class MyUser(HttpUser): + wait_time = between(0, 0.1) + host = "http://192.168.1.112:19122" + + @task + def search(self): + response = self.client.put(url=url, json=data, headers=headers, timeout=2) + print(response) diff --git a/tests/milvus_benchmark/locust_flush_task.py b/tests/milvus_benchmark/locust_flush_task.py new file mode 100644 index 0000000000..be93a189ee --- /dev/null +++ b/tests/milvus_benchmark/locust_flush_task.py @@ -0,0 +1,34 @@ +import random +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient +from milvus import DataType + +connection_type = "single" +host = "192.168.1.6" +port = 19530 +collection_name = "create_collection_CZkkwJgo" +dim = 128 +nb = 50000 +segment_row_limit = 10000 +m = MilvusClient(host=host, port=port, collection_name=collection_name) +m.clean_db() +m.create_collection(dim, segment_row_limit, data_type=DataType.FLOAT_VECTOR, auto_id=True, other_fields=None) +vectors = [[random.random() for _ in range(dim)] for _ in range(nb)] +entities = m.generate_entities(vectors) + + +class FlushTask(User): + wait_time = between(0.001, 0.002) + if connection_type == "single": + client = MilvusTask(m=m) + else: + client = MilvusTask(host=host, port=port, collection_name=collection_name) + + # def insert(self): + # self.client.insert(entities) + + @task(1) + def flush(self): + self.client.insert(entities) + self.client.flush(collection_name) diff --git a/tests/milvus_benchmark/locust_get_entity_task.py b/tests/milvus_benchmark/locust_get_entity_task.py new file mode 100644 index 0000000000..c63e913db4 --- /dev/null +++ b/tests/milvus_benchmark/locust_get_entity_task.py @@ -0,0 +1,37 @@ +import logging +import random +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient +from milvus import DataType + +connection_type = "single" +host = "192.168.1.6" +port = 19530 +collection_name = "sift_10m_100000_128_l2" +dim = 128 +segment_row_limit = 10000 +m = MilvusClient(host=host, port=port, collection_name=collection_name) +# m.clean_db() +# m.create_collection(dim, segment_row_limit, data_type=DataType.FLOAT_VECTOR, auto_id=True, other_fields=None) +nb = 6000 +# vectors = [[random.random() for _ in range(dim)] for _ in range(nb)] +# entities = m.generate_entities(vectors) +ids = [i for i in range(nb)] + +class GetEntityTask(User): + wait_time = between(0.001, 0.002) + if connection_type == "single": + client = MilvusTask(m=m) + else: + client = MilvusTask(host=host, port=port, collection_name=collection_name) + + # def insert(self): + # self.client.insert(entities) + + @task(1) + def get_entity_by_id(self): + # num = random.randint(100, 200) + # get_ids = random.sample(ids, num) + self.client.get_entities([0]) + # logging.getLogger().info(len(get_res)) diff --git a/tests/milvus_benchmark/locust_insert_task.py b/tests/milvus_benchmark/locust_insert_task.py new file mode 100644 index 0000000000..62c9049916 --- /dev/null +++ b/tests/milvus_benchmark/locust_insert_task.py @@ -0,0 +1,35 @@ +import random +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient +from milvus import DataType + +connection_type = "single" +host = "192.168.1.6" +port = 19530 +collection_name = "create_collection_hello" +dim = 128 +nb = 50000 +segment_row_limit = 10000 +m = MilvusClient(host=host, port=port, collection_name=collection_name) +# m.clean_db() +m.create_collection(dim, segment_row_limit, data_type=DataType.FLOAT_VECTOR, auto_id=True, other_fields=None) +vectors = [[random.random() for _ in range(dim)] for _ in range(nb)] +entities = m.generate_entities(vectors) + + +class FlushTask(User): + wait_time = between(0.001, 0.002) + if connection_type == "single": + client = MilvusTask(m=m) + else: + client = MilvusTask(host=host, port=port, collection_name=collection_name) + + @task(1) + def insert(self): + self.client.insert(entities) + # @task(1) + # def create_partition(self): + # tag = 'tag_'.join(random.choice(string.ascii_letters) for _ in range(8)) + # logging.info(tag) + # self.client.create_partition(tag, collection_name) diff --git a/tests/milvus_benchmark/locust_search_task.py b/tests/milvus_benchmark/locust_search_task.py new file mode 100644 index 0000000000..2ac69b8d41 --- /dev/null +++ b/tests/milvus_benchmark/locust_search_task.py @@ -0,0 +1,38 @@ +import logging +import random +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient + +connection_type = "single" +host = "192.168.1.6" +port = 19530 +collection_name = "sift_1m_2000000_128_l2_2" +m = MilvusClient(host=host, port=port, collection_name=collection_name) +dim = 128 +top_k = 10 +nq = 1 +X = [[random.random() for i in range(dim)] for i in range(nq)] +search_params = {"nprobe": 32} +vector_query = {"vector": {'float_vector': { + "topk": top_k, + "query": X, + "params": search_params, + 'metric_type': 'L2'}}} +# m.clean_db() + + +class QueryTask(User): + wait_time = between(0.001, 0.002) + if connection_type == "single": + client = MilvusTask(m=m) + else: + client = MilvusTask(host=host, port=port, collection_name=collection_name) + # client = MilvusClient(collection_name, host, port) + + def preload(self): + self.client.preload_collection() + + @task(10) + def query(self): + self.client.query(vector_query, collection_name=collection_name) \ No newline at end of file diff --git a/tests/milvus_benchmark/locust_task.py b/tests/milvus_benchmark/locust_task.py new file mode 100644 index 0000000000..944c93c002 --- /dev/null +++ b/tests/milvus_benchmark/locust_task.py @@ -0,0 +1,35 @@ +import time +import random +import logging +from locust import User, events +from client import MilvusClient + + +class MilvusTask(object): + def __init__(self, *args, **kwargs): + self.request_type = "grpc" + connection_type = kwargs.get("connection_type") + if connection_type == "single": + self.m = kwargs.get("m") + elif connection_type == "multi": + host = kwargs.get("host") + port = kwargs.get("port") + collection_name = kwargs.get("collection_name") + self.m = MilvusClient(host=host, port=port, collection_name=collection_name) + + def __getattr__(self, name): + func = getattr(self.m, name) + + def wrapper(*args, **kwargs): + start_time = time.time() + try: + result = func(*args, **kwargs) + total_time = int((time.time() - start_time) * 1000) + events.request_success.fire(request_type=self.request_type, name=name, response_time=total_time, + response_length=0) + except Exception as e: + total_time = int((time.time() - start_time) * 1000) + events.request_failure.fire(request_type=self.request_type, name=name, response_time=total_time, + exception=e, response_length=0) + + return wrapper diff --git a/tests/milvus_benchmark/locust_tasks.py b/tests/milvus_benchmark/locust_tasks.py new file mode 100644 index 0000000000..b34bf514d0 --- /dev/null +++ b/tests/milvus_benchmark/locust_tasks.py @@ -0,0 +1,41 @@ +import random +import time +import logging +from locust import TaskSet, task + +dim = 128 + + +class Tasks(TaskSet): + + @task + def query(self): + top_k = 10 + search_param = {"nprobe": 16} + X = [[random.random() for i in range(dim)]] + vector_query = {"vector": {"float_vector": { + "topk": top_k, + "query": X, + "metric_type": "L2", + "params": search_param} + }} + filter_query = None + self.client.query(vector_query, filter_query=filter_query) + + @task + def flush(self): + self.client.flush() + + @task + def compact(self): + self.client.compact() + + @task + def delete(self): + self.client.delete([random.randint(1, 1000000)]) + + @task + def insert(self): + id = random.randint(10000000, 10000000000) + X = [[random.random() for i in range(dim)] for i in range(1)] + self.client.insert(X, ids=[id]) diff --git a/tests/milvus_benchmark/locust_user.py b/tests/milvus_benchmark/locust_user.py new file mode 100644 index 0000000000..e24f68cd56 --- /dev/null +++ b/tests/milvus_benchmark/locust_user.py @@ -0,0 +1,66 @@ +import logging +import random +import pdb +import gevent +import gevent.monkey +gevent.monkey.patch_all() + +from locust import User, between, events, stats +from locust.env import Environment +from locust.stats import stats_printer, print_stats +from locust.log import setup_logging, greenlet_exception_logger + +from locust_tasks import Tasks +from client import MilvusClient +from locust_task import MilvusTask + + +class MyUser(User): + # task_set = None + wait_time = between(0.001, 0.002) + + +def locust_executor(host, port, collection_name, connection_type="single", run_params=None): + m = MilvusClient(host=host, port=port, collection_name=collection_name) + MyUser.tasks = {} + tasks = run_params["tasks"] + for op, weight in tasks.items(): + task = {eval("Tasks."+op): weight} + MyUser.tasks.update(task) + # MyUser.tasks = {Tasks.query: 1, Tasks.flush: 1} + MyUser.client = MilvusTask(host=host, port=port, collection_name=collection_name, connection_type=connection_type, m=m) + env = Environment(events=events, user_classes=[MyUser]) + runner = env.create_local_runner() + # setup logging + logger = logging.getLogger("__locust__") + setup_logging("INFO", None) + greenlet_exception_logger(logger=logger) + gevent.spawn(stats_printer(env.stats)) + # env.create_web_ui("127.0.0.1", 8089) + # gevent.spawn(stats_printer(env.stats), env, "test", full_history=True) + # events.init.fire(environment=env, runner=runner) + clients_num = run_params["clients_num"] + spawn_rate = run_params["spawn_rate"] + during_time = run_params["during_time"] + runner.start(clients_num, spawn_rate=spawn_rate) + gevent.spawn_later(during_time, lambda: runner.quit()) + runner.greenlet.join() + print_stats(env.stats) + result = { + "rps": round(env.stats.total.current_rps, 1), + "fail_ratio": env.stats.total.fail_ratio, + "max_response_time": round(env.stats.total.max_response_time, 1), + "min_response_time": round(env.stats.total.avg_response_time, 1) + } + logging.info(result) + runner.stop() + return result + + +if __name__ == '__main__': + connection_type = "single" + host = "192.168.1.112" + port = 19530 + collection_name = "sift_1m_2000000_128_l2_2" + run_params = {"tasks": {"query": 1, "flush": 1}, "clients_num": 1, "spawn_rate": 1, "during_time": 3} + locust_executor(host, port, collection_name, run_params=run_params) diff --git a/tests/milvus_benchmark/main.py b/tests/milvus_benchmark/main.py index 7738829785..ba35b5315a 100644 --- a/tests/milvus_benchmark/main.py +++ b/tests/milvus_benchmark/main.py @@ -1,6 +1,7 @@ import os import sys import time +from datetime import datetime import pdb import argparse import logging @@ -17,12 +18,23 @@ import parser DEFAULT_IMAGE = "milvusdb/milvus:latest" LOG_FOLDER = "logs" NAMESPACE = "milvus" +LOG_PATH = "/test/milvus/benchmark/logs/" -logging.basicConfig(format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', - datefmt='%Y-%m-%d:%H:%M:%S', - level=logging.DEBUG) -logger = logging.getLogger("milvus_benchmark") - +logger = logging.getLogger('milvus_benchmark') +logger.setLevel(logging.DEBUG) +# create file handler which logs even debug messages +fh = logging.FileHandler(LOG_PATH+'benchmark-{:%Y-%m-%d}.log'.format(datetime.now())) +fh.setLevel(logging.DEBUG) +# create console handler with a higher log level +ch = logging.StreamHandler() +ch.setLevel(logging.INFO) +# create formatter and add it to the handlers +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +fh.setFormatter(formatter) +ch.setFormatter(formatter) +# add the handlers to the logger +logger.addHandler(fh) +logger.addHandler(ch) def positive_int(s): i = None @@ -38,7 +50,7 @@ def positive_int(s): def get_image_tag(image_version, image_type): return "%s-%s-centos7-release" % (image_version, image_type) # return "%s-%s-centos7-release" % ("0.7.1", image_type) - # return "%s-%s-centos7-release" % ("PR-2159", image_type) + # return "%s-%s-centos7-release" % ("PR-2780", image_type) def queue_worker(queue): @@ -46,6 +58,7 @@ def queue_worker(queue): q = queue.get() suite = q["suite"] server_host = q["server_host"] + deploy_mode = q["deploy_mode"] image_type = q["image_type"] image_tag = q["image_tag"] @@ -58,10 +71,12 @@ def queue_worker(queue): collections = run_params["collections"] for collection in collections: # run tests - server_config = collection["server"] + milvus_config = collection["milvus"] if "milvus" in collection else None + server_config = collection["server"] if "server" in collection else None + logger.debug(milvus_config) logger.debug(server_config) runner = K8sRunner() - if runner.init_env(server_config, server_host, image_type, image_tag): + if runner.init_env(milvus_config, server_config, server_host, deploy_mode, image_type, image_tag): logger.debug("Start run tests") try: runner.run(run_type, collection) @@ -69,10 +84,12 @@ def queue_worker(queue): logger.error(str(e)) logger.error(traceback.format_exc()) finally: + time.sleep(60) runner.clean_up() else: logger.error("Runner init failed") - logger.debug("All task finished in queue: %s" % server_host) + if server_host: + logger.debug("All task finished in queue: %s" % server_host) def main(): @@ -88,6 +105,10 @@ def main(): metavar='FILE', default='', help="load test schedule from FILE") + arg_parser.add_argument( + "--deploy-mode", + default='', + help="single or shards") # local mode arg_parser.add_argument( @@ -116,15 +137,17 @@ def main(): if not args.image_version: raise Exception("Image version not given") image_version = args.image_version + deploy_mode = args.deploy_mode with open(args.schedule_conf) as f: schedule_config = full_load(f) f.close() queues = [] - server_names = set() + # server_names = set() + server_names = [] for item in schedule_config: - server_host = item["server"] + server_host = item["server"] if "server" in item else "" suite_params = item["suite_params"] - server_names.add(server_host) + server_names.append(server_host) q = Queue() for suite_param in suite_params: suite = "suites/"+suite_param["suite"] @@ -133,11 +156,12 @@ def main(): q.put({ "suite": suite, "server_host": server_host, + "deploy_mode": deploy_mode, "image_tag": image_tag, "image_type": image_type }) queues.append(q) - logger.debug(server_names) + logging.error(queues) thread_num = len(server_names) processes = [] @@ -145,10 +169,12 @@ def main(): x = Process(target=queue_worker, args=(queues[i], )) processes.append(x) x.start() - time.sleep(5) + time.sleep(10) for x in processes: x.join() + # queue_worker(queues[0]) + elif args.local: # for local mode host = args.host diff --git a/tests/milvus_benchmark/mix_task.py b/tests/milvus_benchmark/mix_task.py new file mode 100644 index 0000000000..b6242db1f5 --- /dev/null +++ b/tests/milvus_benchmark/mix_task.py @@ -0,0 +1,45 @@ +import random +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient + +connection_type = "single" +host = "192.168.1.29" +port = 19530 +collection_name = "sift_128_euclidean" +dim = 128 +m = MilvusClient(host=host, port=port, collection_name=collection_name) + + +class MixTask(User): + wait_time = between(0.001, 0.002) + print("in query task") + if connection_type == "single": + client = MilvusTask(m=m) + else: + client = MilvusTask(host=host, port=port, collection_name=collection_name) + + @task(30) + def query(self): + top_k = 10 + X = [[random.random() for i in range(dim)] for i in range(1)] + search_param = {"nprobe": 16} + self.client.query(X, top_k, search_param) + + @task(10) + def insert(self): + id = random.randint(10000000, 10000000000) + X = [[random.random() for i in range(dim)] for i in range(1)] + self.client.insert(X, ids=[id]) + + @task(1) + def flush(self): + self.client.flush() + + @task(5) + def delete(self): + self.client.delete([random.randint(1, 1000000)]) + + @task(1) + def compact(self): + self.client.compact() diff --git a/tests/milvus_benchmark/parser.py b/tests/milvus_benchmark/parser.py index 1615e72267..05770593e7 100644 --- a/tests/milvus_benchmark/parser.py +++ b/tests/milvus_benchmark/parser.py @@ -32,7 +32,7 @@ def collection_parser(collection_name): def parse_ann_collection_name(collection_name): data_type = collection_name.split("_")[0] dimension = int(collection_name.split("_")[1]) - metric = collection_name.split("_")[-1] + metric = collection_name.split("_")[2] # metric = collection_name.attrs['distance'] # dimension = len(collection_name["train"][0]) if metric == "euclidean": @@ -43,7 +43,7 @@ def parse_ann_collection_name(collection_name): metric_type = "jaccard" elif metric == "hamming": metric_type = "hamming" - return ("ann_"+data_type, dimension, metric_type) + return (data_type, dimension, metric_type) def search_params_parser(param): @@ -83,4 +83,4 @@ def search_params_parser(param): else: logger.warning("Invalid format nprobes: %s" % str(nprobes)) - return top_ks, nqs, nprobes \ No newline at end of file + return top_ks, nqs, nprobes diff --git a/tests/milvus_benchmark/report.py b/tests/milvus_benchmark/report.py deleted file mode 100644 index 6041311513..0000000000 --- a/tests/milvus_benchmark/report.py +++ /dev/null @@ -1,10 +0,0 @@ -# from tablereport import Table -# from tablereport.shortcut import write_to_excel - -# RESULT_FOLDER = "results" - -# def create_table(headers, bodys, table_name): -# table = Table(header=[headers], -# body=[bodys]) - -# write_to_excel('%s/%s.xlsx' % (RESULT_FOLDER, table_name), table) \ No newline at end of file diff --git a/tests/milvus_benchmark/requirements.txt b/tests/milvus_benchmark/requirements.txt index a9ab1ee045..ce35788c2d 100644 --- a/tests/milvus_benchmark/requirements.txt +++ b/tests/milvus_benchmark/requirements.txt @@ -1,4 +1,4 @@ -pymilvus-test>=0.2.0 +pymilvus-test==0.4.16 scipy==1.3.1 scikit-learn==0.19.1 h5py==2.7.1 @@ -6,6 +6,7 @@ h5py==2.7.1 pyyaml>=5.1 tableprint==0.8.0 ansicolors==1.1.8 -scipy==1.3.1 kubernetes==10.0.1 # rq==1.2.0 +locust>=1.0.2 +pymongo==3.10.0 diff --git a/tests/milvus_benchmark/results/__init__.py b/tests/milvus_benchmark/results/__init__.py new file mode 100644 index 0000000000..81b55da1c5 --- /dev/null +++ b/tests/milvus_benchmark/results/__init__.py @@ -0,0 +1,11 @@ + +class Reporter(object): + def __init__(self): + pass + + def report(self, result): + pass + + +class BaseResult(object): + pass \ No newline at end of file diff --git a/tests/milvus_benchmark/results/reporter.py b/tests/milvus_benchmark/results/reporter.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/milvus_benchmark/runner.py b/tests/milvus_benchmark/runner.py index 3be7d712f9..09b43025fc 100644 --- a/tests/milvus_benchmark/runner.py +++ b/tests/milvus_benchmark/runner.py @@ -1,12 +1,15 @@ import os +import threading import logging import pdb import time import random +import grpc from multiprocessing import Process from itertools import product import numpy as np import sklearn.preprocessing +from milvus import DataType from client import MilvusClient import utils import parser @@ -15,7 +18,7 @@ logger = logging.getLogger("milvus_benchmark.runner") VECTORS_PER_FILE = 1000000 SIFT_VECTORS_PER_FILE = 100000 -JACCARD_VECTORS_PER_FILE = 2000000 +BINARY_VECTORS_PER_FILE = 2000000 MAX_NQ = 10001 FILE_PREFIX = "binary_" @@ -24,9 +27,7 @@ FILE_PREFIX = "binary_" SRC_BINARY_DATA_DIR = '/test/milvus/raw_data/random/' SIFT_SRC_DATA_DIR = '/test/milvus/raw_data/sift1b/' DEEP_SRC_DATA_DIR = '/test/milvus/raw_data/deep1b/' -JACCARD_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/' -HAMMING_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/' -STRUCTURE_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/' +BINARY_SRC_DATA_DIR = '/test/milvus/raw_data/binary/' SIFT_SRC_GROUNDTRUTH_DATA_DIR = SIFT_SRC_DATA_DIR + 'gnd' WARM_TOP_K = 1 @@ -57,12 +58,8 @@ def gen_file_name(idx, dimension, data_type): fname = SIFT_SRC_DATA_DIR+fname elif data_type == "deep": fname = DEEP_SRC_DATA_DIR+fname - elif data_type == "jaccard": - fname = JACCARD_SRC_DATA_DIR+fname - elif data_type == "hamming": - fname = HAMMING_SRC_DATA_DIR+fname - elif data_type == "sub" or data_type == "super": - fname = STRUCTURE_SRC_DATA_DIR+fname + elif data_type == "binary": + fname = BINARY_SRC_DATA_DIR+fname return fname @@ -76,12 +73,8 @@ def get_vectors_from_binary(nq, dimension, data_type): file_name = SIFT_SRC_DATA_DIR+'query.npy' elif data_type == "deep": file_name = DEEP_SRC_DATA_DIR+'query.npy' - elif data_type == "jaccard": - file_name = JACCARD_SRC_DATA_DIR+'query.npy' - elif data_type == "hamming": - file_name = HAMMING_SRC_DATA_DIR+'query.npy' - elif data_type == "sub" or data_type == "super": - file_name = STRUCTURE_SRC_DATA_DIR+'query.npy' + elif data_type == "binary": + file_name = BINARY_SRC_DATA_DIR+'query.npy' data = np.load(file_name) vectors = data[0:nq].tolist() return vectors @@ -91,6 +84,32 @@ class Runner(object): def __init__(self): pass + def gen_executors(self, operations): + l = [] + for name, operation in operations.items(): + weight = operation["weight"] if "weight" in operation else 1 + l.extend([name] * weight) + random.shuffle(l) + return l + + def get_vector_type(self, data_type): + vector_type = '' + if data_type in ["random", "sift", "deep", "glove"]: + vector_type = DataType.FLOAT_VECTOR + elif data_type in ["binary"]: + vector_type = DataType.BINARY_VECTOR + else: + raise Exception("Data type: %s not defined" % data_type) + return vector_type + + def get_vector_type_from_metric(self, metric_type): + vector_type = '' + if metric_type in ["hamming", "jaccard"]: + vector_type = DataType.BINARY_VECTOR + else: + vector_type = DataType.FLOAT_VECTOR + return vector_type + def normalize(self, metric_type, X): if metric_type == "ip": logger.info("Set normalize for metric_type: %s" % metric_type) @@ -100,7 +119,7 @@ class Runner(object): X = X.astype(np.float32) elif metric_type in ["jaccard", "hamming", "sub", "super"]: tmp = [] - for index, item in enumerate(X): + for item in X: new_vector = bytes(np.packbits(item, axis=-1).tolist()) tmp.append(new_vector) X = tmp @@ -148,56 +167,111 @@ class Runner(object): vectors_per_file = 10000 elif data_type == "sift": vectors_per_file = SIFT_VECTORS_PER_FILE - elif data_type in ["jaccard", "hamming", "sub", "super"]: - vectors_per_file = JACCARD_VECTORS_PER_FILE + elif data_type in ["binary"]: + vectors_per_file = BINARY_VECTORS_PER_FILE else: raise Exception("data_type: %s not supported" % data_type) - if size % vectors_per_file or ni > vectors_per_file: + if size % vectors_per_file or size % ni: raise Exception("Not invalid collection size or ni") - file_num = size // vectors_per_file - for i in range(file_num): - file_name = gen_file_name(i, dimension, data_type) - # logger.info("Load npy file: %s start" % file_name) - data = np.load(file_name) - # logger.info("Load npy file: %s end" % file_name) - loops = vectors_per_file // ni - for j in range(loops): - vectors = data[j*ni:(j+1)*ni].tolist() + i = 0 + while i < (size // vectors_per_file): + vectors = [] + if vectors_per_file >= ni: + file_name = gen_file_name(i, dimension, data_type) + # logger.info("Load npy file: %s start" % file_name) + data = np.load(file_name) + # logger.info("Load npy file: %s end" % file_name) + for j in range(vectors_per_file // ni): + vectors = data[j*ni:(j+1)*ni].tolist() + if vectors: + # start insert vectors + start_id = i * vectors_per_file + j * ni + end_id = start_id + len(vectors) + logger.info("Start id: %s, end id: %s" % (start_id, end_id)) + ids = [k for k in range(start_id, end_id)] + entities = milvus.generate_entities(vectors, ids) + ni_start_time = time.time() + try: + res_ids = milvus.insert(entities, ids=ids) + except grpc.RpcError as e: + if e.code() == grpc.StatusCode.UNAVAILABLE: + logger.debug("Retry insert") + def retry(): + res_ids = milvus.insert(entities, ids=ids) + + t0 = threading.Thread(target=retry) + t0.start() + t0.join() + logger.debug("Retry successfully") + raise e + assert ids == res_ids + # milvus.flush() + logger.debug(milvus.count()) + ni_end_time = time.time() + total_time = total_time + ni_end_time - ni_start_time + i += 1 + else: + vectors.clear() + loops = ni // vectors_per_file + for j in range(loops): + file_name = gen_file_name(loops*i+j, dimension, data_type) + data = np.load(file_name) + vectors.extend(data.tolist()) if vectors: - ni_start_time = time.time() - # start insert vectors - start_id = i * vectors_per_file + j * ni + start_id = i * vectors_per_file end_id = start_id + len(vectors) logger.info("Start id: %s, end id: %s" % (start_id, end_id)) ids = [k for k in range(start_id, end_id)] - status, ids = milvus.insert(vectors, ids=ids) + entities = milvus.generate_entities(vectors, ids) + ni_start_time = time.time() + try: + res_ids = milvus.insert(entities, ids=ids) + except grpc.RpcError as e: + if e.code() == grpc.StatusCode.UNAVAILABLE: + logger.debug("Retry insert") + def retry(): + res_ids = milvus.insert(entities, ids=ids) + + t0 = threading.Thread(target=retry) + t0.start() + t0.join() + logger.debug("Retry successfully") + raise e + + assert ids == res_ids # milvus.flush() logger.debug(milvus.count()) ni_end_time = time.time() total_time = total_time + ni_end_time - ni_start_time - + i += loops qps = round(size / total_time, 2) - ni_time = round(total_time / (loops * file_num), 2) + ni_time = round(total_time / (size / ni), 2) bi_res["total_time"] = round(total_time, 2) bi_res["qps"] = qps bi_res["ni_time"] = ni_time return bi_res - def do_query(self, milvus, collection_name, top_ks, nqs, run_count=1, search_param=None): + def do_query(self, milvus, collection_name, vec_field_name, top_ks, nqs, run_count=1, search_param=None, filter_query=None): bi_res = [] (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type) for nq in nqs: tmp_res = [] - vectors = base_query_vectors[0:nq] + query_vectors = base_query_vectors[0:nq] for top_k in top_ks: avg_query_time = 0.0 min_query_time = 0.0 - logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors))) + logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(query_vectors))) for i in range(run_count): logger.info("Start run query, run %d of %s" % (i+1, run_count)) start_time = time.time() - query_res = milvus.query(vectors, top_k, search_param=search_param) + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors, + "metric_type": utils.metric_type_trans(metric_type), + "params": search_param} + }} + query_res = milvus.query(vector_query, filter_query=filter_query) interval_time = time.time() - start_time if (i == 0) or (min_query_time > interval_time): min_query_time = interval_time @@ -212,23 +286,20 @@ class Runner(object): end_time = time.time() return end_time - start_time - def do_query_ids(self, milvus, collection_name, top_k, nq, search_param=None): + def do_query_ids(self, milvus, collection_name, vec_field_name, top_k, nq, search_param=None, filter_query=None): (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type) - vectors = base_query_vectors[0:nq] - logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors))) - query_res = milvus.query(vectors, top_k, search_param=search_param) - result_ids = [] - result_distances = [] - for result in query_res: - tmp = [] - tmp_distance = [] - for item in result: - tmp.append(item.id) - tmp_distance.append(item.distance) - result_ids.append(tmp) - result_distances.append(tmp_distance) - return result_ids, result_distances + query_vectors = base_query_vectors[0:nq] + logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(query_vectors))) + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors, + "metric_type": utils.metric_type_trans(metric_type), + "params": search_param} + }} + query_res = milvus.query(vector_query, filter_query=filter_query) + result_ids = milvus.get_ids(query_res) + return result_ids def do_query_acc(self, milvus, collection_name, top_k, nq, id_store_name, search_param=None): (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) @@ -286,3 +357,13 @@ class Runner(object): d = a[0] true_ids = a.reshape(-1, d + 1)[:, 1:].copy() return true_ids + + def get_fields(self, milvus, collection_name): + fields = [] + info = milvus.get_info(collection_name) + for item in info["fields"]: + fields.append(item["name"]) + return fields + + # def get_filter_query(self, filter_query): + # for filter in filter_query: diff --git a/tests/milvus_benchmark/runners/__init__.py b/tests/milvus_benchmark/runners/__init__.py new file mode 100644 index 0000000000..4b082462d5 --- /dev/null +++ b/tests/milvus_benchmark/runners/__init__.py @@ -0,0 +1,11 @@ + + +class BaseRunner(object): + def __init__(self): + pass + + def set_up(self): + pass + + def tear_down(self): + pass diff --git a/tests/milvus_benchmark/runners/locust_runner.py b/tests/milvus_benchmark/runners/locust_runner.py new file mode 100644 index 0000000000..afe2b1e043 --- /dev/null +++ b/tests/milvus_benchmark/runners/locust_runner.py @@ -0,0 +1,75 @@ +import time +import random +from locust import Locust, TaskSet, events, task, between +from client import MilvusClient +from . import BasicRunner + + +dim = 128 +top_k = 10 +X = [[random.random() for i in range(dim)] for i in range(1)] +search_param = {"nprobe": 16} + + +class MilvusTask(object): + def __init__(self, type="single", args): + self.type = type + self.m = None + if type == "single": + self.m = MilvusClient(host=args["host"], port=args["port"], collection_name=args["collection_name"]) + elif type == "multi": + self.m = MilvusClient(host=args["m"]) + + def query(self, *args, **kwargs): + name = "milvus_search" + request_type = "grpc" + start_time = time.time() + try: + # result = self.m.getattr(*args, **kwargs) + status, result = self.m.query(*args, **kwargs) + except Exception as e: + total_time = int((time.time() - start_time) * 1000) + events.request_failure.fire(request_type=request_type, name=name, response_time=total_time, exception=e, response_length=0) + else: + if not status.OK: + total_time = int((time.time() - start_time) * 1000) + events.request_failure.fire(request_type=request_type, name=name, response_time=total_time, exception=e, response_length=0) + else: + total_time = int((time.time() - start_time) * 1000) + events.request_success.fire(request_type=request_type, name=name, response_time=total_time, response_length=0) + # In this example, I've hardcoded response_length=0. If we would want the response length to be + # reported correctly in the statistics, we would probably need to hook in at a lower level + + +class MilvusLocust(Locust): + def __init__(self, *args, **kwargs): + super(MilvusLocust, self).__init__(*args, **kwargs) + self.client = MilvusTask(self.host, self.port, self.collection_name) + + +class Query(MilvusLocust): + host = "192.168.1.183" + port = 19530 + collection_name = "sift_128_euclidean" + # m = MilvusClient(host=host, port=port, collection_name=collection_name) + wait_time = between(0.001, 0.002) + + class task_set(TaskSet): + @task + def query(self): + self.client.query(X, top_k, search_param) + + +class LocustRunner(BasicRunner): + """Only one client, not support M/S mode""" + def __init__(self, args): + # Start client with params including client number && last time && hatch rate ... + pass + + def set_up(self): + # helm install locust client + pass + + def tear_down(self): + # helm uninstall + pass diff --git a/tests/milvus_benchmark/scheduler/010_data.json b/tests/milvus_benchmark/scheduler/010_data.json new file mode 100644 index 0000000000..d7074b63f5 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/010_data.json @@ -0,0 +1,65 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "080_gpu_accuracy.yaml", + "image_type": "gpu" + }, + { + "suite": "080_search_stability.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_accuracy_ann.yaml", + "image_type": "gpu" + } + ] + }, + { + "server": "poseidon", + "suite_params": [ + { + "suite": "080_gpu_search.yaml", + "image_type": "gpu" + }, + { + "suite": "080_cpu_search.yaml", + "image_type": "cpu" + }, + { + "suite": "080_gpu_build.yaml", + "image_type": "gpu" + }, + { + "suite": "080_cpu_accuracy.yaml", + "image_type": "cpu" + }, + { + "suite": "locust_search.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "apollo", + "suite_params": [ + { + "suite": "cpu_accuracy_ann.yaml", + "image_type": "cpu" + }, + { + "suite": "080_cpu_build.yaml", + "image_type": "cpu" + }, + { + "suite": "080_insert_performance.yaml", + "image_type": "cpu" + }, + { + "suite": "add_flush_performance.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/011_data.json b/tests/milvus_benchmark/scheduler/011_data.json new file mode 100644 index 0000000000..5f301a24bd --- /dev/null +++ b/tests/milvus_benchmark/scheduler/011_data.json @@ -0,0 +1,53 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "011_cpu_accuracy_ann.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "eros", + "suite_params": [ + { + "suite": "locust_mix.yaml", + "image_type": "gpu" + }, + { + "suite": "011_gpu_accuracy.yaml", + "image_type": "gpu" + } + ] + }, + { + "server": "poseidon", + "suite_params": [ + { + "suite": "011_gpu_search.yaml", + "image_type": "gpu" + }, + { + "suite": "011_cpu_search.yaml", + "image_type": "cpu" + }, + { + "suite": "011_gpu_build.yaml", + "image_type": "gpu" + }, + { + "suite": "011_cpu_accuracy.yaml", + "image_type": "cpu" + }, + { + "suite": "011_locust_search.yaml", + "image_type": "cpu" + }, + { + "suite": "011_gpu_accuracy_ann.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/011_data_acc_debug.json b/tests/milvus_benchmark/scheduler/011_data_acc_debug.json new file mode 100644 index 0000000000..3bb0df2259 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/011_data_acc_debug.json @@ -0,0 +1,11 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "011_cpu_accuracy_ann.yaml", + "image_type": "cpu" + } + ] + } +] \ No newline at end of file diff --git a/tests/milvus_benchmark/scheduler/ann_acc.json b/tests/milvus_benchmark/scheduler/011_data_gpu_build.json similarity index 69% rename from tests/milvus_benchmark/scheduler/ann_acc.json rename to tests/milvus_benchmark/scheduler/011_data_gpu_build.json index 32cc93604c..ed9642fa91 100644 --- a/tests/milvus_benchmark/scheduler/ann_acc.json +++ b/tests/milvus_benchmark/scheduler/011_data_gpu_build.json @@ -1,11 +1,11 @@ [ - { + { "server": "eros", "suite_params": [ { - "suite": "070_ann.yaml", + "suite": "011_gpu_build_sift10m.yaml", "image_type": "gpu" } ] } -] +] \ No newline at end of file diff --git a/tests/milvus_benchmark/scheduler/011_data_insert.json b/tests/milvus_benchmark/scheduler/011_data_insert.json new file mode 100644 index 0000000000..c8bb875ac9 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/011_data_insert.json @@ -0,0 +1,11 @@ +[ + { + "server": "eros", + "suite_params": [ + { + "suite": "011_insert_data.yaml", + "image_type": "cpu" + } + ] + } + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/scheduler/011_data_search_debug.json b/tests/milvus_benchmark/scheduler/011_data_search_debug.json new file mode 100644 index 0000000000..dec44ac097 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/011_data_search_debug.json @@ -0,0 +1,11 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "011_gpu_search_debug.yaml", + "image_type": "gpu" + } + ] + } +] \ No newline at end of file diff --git a/tests/milvus_benchmark/scheduler/011_delete.json b/tests/milvus_benchmark/scheduler/011_delete.json new file mode 100644 index 0000000000..cc80004991 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/011_delete.json @@ -0,0 +1,15 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "011_insert_performance.yaml", + "image_type": "cpu" + }, + { + "suite": "011_delete_performance.yaml", + "image_type": "cpu" + } + ] + } +] \ No newline at end of file diff --git a/tests/milvus_benchmark/scheduler/080_data.json b/tests/milvus_benchmark/scheduler/080_data.json index 3c375aee62..d7074b63f5 100644 --- a/tests/milvus_benchmark/scheduler/080_data.json +++ b/tests/milvus_benchmark/scheduler/080_data.json @@ -5,6 +5,14 @@ { "suite": "080_gpu_accuracy.yaml", "image_type": "gpu" + }, + { + "suite": "080_search_stability.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_accuracy_ann.yaml", + "image_type": "gpu" } ] }, @@ -28,7 +36,7 @@ "image_type": "cpu" }, { - "suite": "080_cpu_build.yaml", + "suite": "locust_search.yaml", "image_type": "cpu" } ] @@ -41,23 +49,17 @@ "image_type": "cpu" }, { - "suite": "080_cpu_search_stability.yaml", + "suite": "080_cpu_build.yaml", + "image_type": "cpu" + }, + { + "suite": "080_insert_performance.yaml", + "image_type": "cpu" + }, + { + "suite": "add_flush_performance.yaml", "image_type": "cpu" } ] - }, - { - "server": "eros", - "suite_params": [ - { - "suite": "gpu_accuracy_ann.yaml", - "image_type": "gpu" - }, - { - "suite": "080_gpu_stability.yaml", - "image_type": "gpu" - } - ] } - ] diff --git a/tests/milvus_benchmark/scheduler/apollo.json b/tests/milvus_benchmark/scheduler/apollo.json index 6d2c529c1c..97761f5d04 100644 --- a/tests/milvus_benchmark/scheduler/apollo.json +++ b/tests/milvus_benchmark/scheduler/apollo.json @@ -3,11 +3,7 @@ "server": "apollo", "suite_params": [ { - "suite": "070_insert_10m.yaml", - "image_type": "cpu" - }, - { - "suite": "070_cpu_build.yaml", + "suite": "011_cpu_accuracy_ann.yaml", "image_type": "cpu" } ] diff --git a/tests/milvus_benchmark/scheduler/athena.json b/tests/milvus_benchmark/scheduler/athena.json index c9b3b47eb8..adc6528678 100644 --- a/tests/milvus_benchmark/scheduler/athena.json +++ b/tests/milvus_benchmark/scheduler/athena.json @@ -3,8 +3,16 @@ "server": "athena", "suite_params": [ { - "suite": "add_flush_performance.yaml", - "image_type": "cpu" + "suite": "011_gpu_accuracy.yaml", + "image_type": "gpu" + }, + { + "suite": "011_search_stability.yaml", + "image_type": "gpu" + }, + { + "suite": "011_gpu_accuracy_ann.yaml", + "image_type": "gpu" } ] } diff --git a/tests/milvus_benchmark/scheduler/build.json b/tests/milvus_benchmark/scheduler/build.json index d5961c2ed4..f269669e1f 100644 --- a/tests/milvus_benchmark/scheduler/build.json +++ b/tests/milvus_benchmark/scheduler/build.json @@ -1,9 +1,9 @@ [ { - "server": "athena", + "server": "eros", "suite_params": [ { - "suite": "debug_build.yaml", + "suite": "011_gpu_build_sift1b.yaml", "image_type": "gpu" } ] diff --git a/tests/milvus_benchmark/scheduler/crud_flush.json b/tests/milvus_benchmark/scheduler/cqx.json similarity index 74% rename from tests/milvus_benchmark/scheduler/crud_flush.json rename to tests/milvus_benchmark/scheduler/cqx.json index 44c52eadeb..610f5a2cf5 100644 --- a/tests/milvus_benchmark/scheduler/crud_flush.json +++ b/tests/milvus_benchmark/scheduler/cqx.json @@ -3,7 +3,7 @@ "server": "apollo", "suite_params": [ { - "suite": "crud_add_flush.yaml", + "suite": "011_insert_debug.yaml", "image_type": "cpu" } ] diff --git a/tests/milvus_benchmark/scheduler/crud_search.json b/tests/milvus_benchmark/scheduler/crud_search.json deleted file mode 100644 index ccb95a7d28..0000000000 --- a/tests/milvus_benchmark/scheduler/crud_search.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "server": "athena", - "suite_params": [ - { - "suite": "crud_search.yaml", - "image_type": "gpu" - } - ] - } -] diff --git a/tests/milvus_benchmark/scheduler/debug.json b/tests/milvus_benchmark/scheduler/debug.json index 27ed229fd9..be9cbbe231 100644 --- a/tests/milvus_benchmark/scheduler/debug.json +++ b/tests/milvus_benchmark/scheduler/debug.json @@ -3,8 +3,8 @@ "server": "poseidon", "suite_params": [ { - "suite": "debug.yaml", - "image_type": "gpu" + "suite": "clean.yaml", + "image_type": "cpu" } ] } diff --git a/tests/milvus_benchmark/scheduler/eros.json b/tests/milvus_benchmark/scheduler/eros.json index 58ccc78a33..dd74aa188a 100644 --- a/tests/milvus_benchmark/scheduler/eros.json +++ b/tests/milvus_benchmark/scheduler/eros.json @@ -1,10 +1,10 @@ [ { - "server": "eros", + "server": "poseidon", "suite_params": [ { - "suite": "080_gpu_search_id.yaml", - "image_type": "gpu" + "suite": "011_locust_search.yaml", + "image_type": "cpu" } ] } diff --git a/tests/milvus_benchmark/scheduler/crud_build.json b/tests/milvus_benchmark/scheduler/filter.json similarity index 75% rename from tests/milvus_benchmark/scheduler/crud_build.json rename to tests/milvus_benchmark/scheduler/filter.json index 69efcc81be..2a0baed660 100644 --- a/tests/milvus_benchmark/scheduler/crud_build.json +++ b/tests/milvus_benchmark/scheduler/filter.json @@ -3,10 +3,9 @@ "server": "poseidon", "suite_params": [ { - "suite": "070_gpu_build.yaml", + "suite": "011_search_dsl.yaml", "image_type": "gpu" } ] } ] - diff --git a/tests/milvus_benchmark/scheduler/insert.json b/tests/milvus_benchmark/scheduler/insert.json index c6c23a376e..fc8aa4ee40 100644 --- a/tests/milvus_benchmark/scheduler/insert.json +++ b/tests/milvus_benchmark/scheduler/insert.json @@ -1,10 +1,9 @@ [ { - "server": "poseidon", "suite_params": [ { - "suite": "insert_performance.yaml", - "image_type": "gpu" + "suite": "080_insert_performance.yaml", + "image_type": "cpu" } ] } diff --git a/tests/milvus_benchmark/scheduler/jaccard.json b/tests/milvus_benchmark/scheduler/jaccard.json index 9c73c4f0b5..5d5ebd6d13 100644 --- a/tests/milvus_benchmark/scheduler/jaccard.json +++ b/tests/milvus_benchmark/scheduler/jaccard.json @@ -1,9 +1,9 @@ [ { - "server": "poseidon", + "server": "athena", "suite_params": [ { - "suite": "cpu_search_performance_jaccard.yaml", + "suite": "011_cpu_search_binary.yaml", "image_type": "cpu" } ] diff --git a/tests/milvus_benchmark/scheduler/crud_add.json b/tests/milvus_benchmark/scheduler/locust.json similarity index 74% rename from tests/milvus_benchmark/scheduler/crud_add.json rename to tests/milvus_benchmark/scheduler/locust.json index 485bf23e66..dd74aa188a 100644 --- a/tests/milvus_benchmark/scheduler/crud_add.json +++ b/tests/milvus_benchmark/scheduler/locust.json @@ -3,7 +3,7 @@ "server": "poseidon", "suite_params": [ { - "suite": "crud_add.yaml", + "suite": "011_locust_search.yaml", "image_type": "cpu" } ] diff --git a/tests/milvus_benchmark/scheduler/hnsw.json b/tests/milvus_benchmark/scheduler/locust_mix_debug.json similarity index 56% rename from tests/milvus_benchmark/scheduler/hnsw.json rename to tests/milvus_benchmark/scheduler/locust_mix_debug.json index cdd8bd6db1..343d429338 100644 --- a/tests/milvus_benchmark/scheduler/hnsw.json +++ b/tests/milvus_benchmark/scheduler/locust_mix_debug.json @@ -1,9 +1,8 @@ [ { - "server": "poseidon", "suite_params": [ { - "suite": "cpu_accuracy_ann_crud_debug.yaml", + "suite": "locust_mix.yaml", "image_type": "cpu" } ] diff --git a/tests/milvus_benchmark/scheduler/crud_stability.json b/tests/milvus_benchmark/scheduler/loop.json similarity index 61% rename from tests/milvus_benchmark/scheduler/crud_stability.json rename to tests/milvus_benchmark/scheduler/loop.json index 74decfae4e..02fc1e261d 100644 --- a/tests/milvus_benchmark/scheduler/crud_stability.json +++ b/tests/milvus_benchmark/scheduler/loop.json @@ -1,9 +1,8 @@ [ { - "server": "eros", "suite_params": [ { - "suite": "070_stability.yaml", + "suite": "loop_stability.yaml", "image_type": "gpu" } ] diff --git a/tests/milvus_benchmark/scheduler/poseidon.json b/tests/milvus_benchmark/scheduler/poseidon.json index 1429c77a57..8ca4798a06 100644 --- a/tests/milvus_benchmark/scheduler/poseidon.json +++ b/tests/milvus_benchmark/scheduler/poseidon.json @@ -3,23 +3,7 @@ "server": "poseidon", "suite_params": [ { - "suite": "080_gpu_search.yaml", - "image_type": "gpu" - }, - { - "suite": "080_cpu_search.yaml", - "image_type": "cpu" - }, - { - "suite": "080_gpu_build.yaml", - "image_type": "gpu" - }, - { - "suite": "080_cpu_accuracy.yaml", - "image_type": "cpu" - }, - { - "suite": "080_cpu_build.yaml", + "suite": "011_cpu_search_binary.yaml", "image_type": "cpu" } ] diff --git a/tests/milvus_benchmark/scheduler/search.json b/tests/milvus_benchmark/scheduler/search.json index 8fb2fef45a..68dc7776ab 100644 --- a/tests/milvus_benchmark/scheduler/search.json +++ b/tests/milvus_benchmark/scheduler/search.json @@ -1,9 +1,9 @@ [ { - "server": "eros", + "server": "athena", "suite_params": [ { - "suite": "debug.yaml", + "suite": "011_cpu_search_sift1b.yaml", "image_type": "cpu" } ] diff --git a/tests/milvus_benchmark/scheduler/search_performance.json b/tests/milvus_benchmark/scheduler/search_performance.json deleted file mode 100644 index 826818e08f..0000000000 --- a/tests/milvus_benchmark/scheduler/search_performance.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "server": "athena", - "suite_params": [ - { - "suite": "070_gpu_search.yaml", - "image_type": "gpu" - } - ] - } -] diff --git a/tests/milvus_benchmark/scheduler/shards.json b/tests/milvus_benchmark/scheduler/shards.json new file mode 100644 index 0000000000..2a9bb4352a --- /dev/null +++ b/tests/milvus_benchmark/scheduler/shards.json @@ -0,0 +1,18 @@ +[ + { + "suite_params": [ + { + "suite": "shards_insert_performance.yaml", + "image_type": "cpu" + }, + { + "suite": "shards_ann_debug.yaml", + "image_type": "cpu" + }, + { + "suite": "shards_loop_stability.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/shards_ann.json b/tests/milvus_benchmark/scheduler/shards_ann.json new file mode 100644 index 0000000000..6282c234cc --- /dev/null +++ b/tests/milvus_benchmark/scheduler/shards_ann.json @@ -0,0 +1,10 @@ +[ + { + "suite_params": [ + { + "suite": "shards_ann_debug.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/shards_debug.json b/tests/milvus_benchmark/scheduler/shards_debug.json new file mode 100644 index 0000000000..9e79ee7758 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/shards_debug.json @@ -0,0 +1,15 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "shards_insert_performance_sift1m.yaml", + "image_type": "cpu" + }, + { + "suite": "shards_search_performance_sift1m.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/shards_stability.json b/tests/milvus_benchmark/scheduler/shards_stability.json new file mode 100644 index 0000000000..c2b71d9472 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/shards_stability.json @@ -0,0 +1,10 @@ +[ + { + "suite_params": [ + { + "suite": "shards_loop_stability.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/yx.json b/tests/milvus_benchmark/scheduler/yx.json new file mode 100644 index 0000000000..1ac409cc1c --- /dev/null +++ b/tests/milvus_benchmark/scheduler/yx.json @@ -0,0 +1,11 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "011_gpu_sift50m_ivf.yaml", + "image_type": "gpu" + } + ] + } + ] \ No newline at end of file diff --git a/tests/milvus_benchmark/scripts/default_config.json b/tests/milvus_benchmark/scripts/default_config.json deleted file mode 100644 index f63b3d746b..0000000000 --- a/tests/milvus_benchmark/scripts/default_config.json +++ /dev/null @@ -1,86 +0,0 @@ -[ - { - "job_name": "milvus-apollo", - "build_params": { - "SUITE": "cpu_accuracy_ann.yaml", - "IMAGE_TYPE": "cpu", - "IMAGE_VERSION": "master", - "SERVER_HOST": "apollo" - } - }, - { - "job_name": "milvus-apollo", - "build_params": { - "SUITE": "cpu_stability_sift50m.yaml", - "IMAGE_TYPE": "cpu", - "IMAGE_VERSION": "master", - "SERVER_HOST": "apollo" - } - }, - - { - "job_name": "milvus-eros", - "build_params": { - "SUITE": "gpu_accuracy_ann.yaml", - "IMAGE_TYPE": "gpu", - "IMAGE_VERSION": "master", - "SERVER_HOST": "eros" - } - }, - { - "job_name": "milvus-eros", - "build_params": { - "SUITE": "gpu_search_stability.yaml", - "IMAGE_TYPE": "gpu", - "IMAGE_VERSION": "master", - "SERVER_HOST": "eros" - } - }, - { - "job_name": "milvus-eros", - "build_params": { - "SUITE": "gpu_build_performance.yaml", - "IMAGE_TYPE": "gpu", - "IMAGE_VERSION": "master", - "SERVER_HOST": "eros" - } - }, - - { - "job_name": "milvus-poseidon", - "build_params": { - "SUITE": "gpu_search_performance.yaml", - "IMAGE_TYPE": "gpu", - "IMAGE_VERSION": "master", - "SERVER_HOST": "poseidon" - } - }, - { - "job_name": "milvus-poseidon", - "build_params": { - "SUITE": "cpu_search_performance.yaml", - "IMAGE_TYPE": "cpu", - "IMAGE_VERSION": "master", - "SERVER_HOST": "poseidon" - } - }, - { - "job_name": "milvus-poseidon", - "build_params": { - "SUITE": "insert_performance.yaml", - "IMAGE_TYPE": "gpu", - "IMAGE_VERSION": "master", - "SERVER_HOST": "poseidon" - } - }, - { - "job_name": "milvus-poseidon", - "build_params": { - "SUITE": "gpu_accuracy.yaml", - "IMAGE_TYPE": "gpu", - "IMAGE_VERSION": "master", - "SERVER_HOST": "poseidon" - } - } - -] \ No newline at end of file diff --git a/tests/milvus_benchmark/scripts/scheduler.py b/tests/milvus_benchmark/scripts/scheduler.py deleted file mode 100644 index 613397a8c5..0000000000 --- a/tests/milvus_benchmark/scripts/scheduler.py +++ /dev/null @@ -1,28 +0,0 @@ -import json -from pprint import pprint -import jenkins - -JENKINS_URL = "****" -server = jenkins.Jenkins(JENKINS_URL, username='****', password='****') -user = server.get_whoami() -version = server.get_version() -print('Hello %s from Jenkins %s' % (user['fullName'], version)) - - -# print(job_config) -# build_params = { -# "SUITE": "gpu_accuracy_ann_debug.yaml", -# "IMAGE_TYPE": "cpu", -# "IMAGE_VERSION": "tanimoto_distance", -# "SERVER_HOST": "eros" -# } -# print(server.build_job(job_name, build_params)) - - -with open("default_config.json") as json_file: - data = json.load(json_file) - for config in data: - build_params = config["build_params"] - job_name = config["job_name"] - res = server.build_job(job_name, build_params) - print(job_name, res) diff --git a/tests/milvus_benchmark/search_task.py b/tests/milvus_benchmark/search_task.py new file mode 100644 index 0000000000..edab0b410d --- /dev/null +++ b/tests/milvus_benchmark/search_task.py @@ -0,0 +1,49 @@ +import random +import logging +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient +from milvus import DataType +import utils + +connection_type = "single" +host = "192.168.1.112" +port = 19530 +collection_name = "sift_10m_100000_128_l2" +dim = 128 +segment_row_limit = 10000 +m = MilvusClient(host=host, port=port, collection_name=collection_name) +# m.clean_db() +# m.create_collection(dim, segment_row_limit, data_type=DataType.FLOAT_VECTOR, auto_id=True, other_fields=None) +vectors = [[random.random() for _ in range(dim)] for _ in range(1000)] +entities = m.generate_entities(vectors) +ids = [i for i in range(10000000)] + +class QueryTask(User): + wait_time = between(0.001, 0.002) + if connection_type == "single": + client = MilvusTask(m=m) + else: + client = MilvusTask(host=host, port=port, collection_name=collection_name) + + # def query(self): + # top_k = 10 + # X = [[random.random() for i in range(dim)] for i in range(1)] + # search_param = {"nprobe": 16} + # self.client.query(X, top_k, search_param) + + + @task(1) + def insert(self): + self.client.insert(entities) + + # @task(1) + # def create(self): + # collection_name = utils.get_unique_name(prefix="locust") + # self.client.create_collection(dim, segment_row_limit, data_type=DataType.FLOAT_VECTOR, auto_id=True, collection_name=collection_name, other_fields=None) + + # @task(1) + # def delete(self): + # delete_ids = random.sample(ids, 100) + # logging.error(delete_ids) + # self.client.delete(delete_ids) \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_add_flush_performance.yaml b/tests/milvus_benchmark/suites/011_add_flush_performance.yaml new file mode 100644 index 0000000000..a3effed27f --- /dev/null +++ b/tests/milvus_benchmark/suites/011_add_flush_performance.yaml @@ -0,0 +1,20 @@ +insert_flush_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_2m_128_128_l2_flush + cache_config.cpu_cache_capacity: 8 + cache_config.insert_buffer_size: 2 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + db_config.auto_flush_interval: 300 + collection_name: sift_2m_128_128_l2 + ni_per: 100000 diff --git a/tests/milvus_benchmark/suites/011_build_debug.yaml b/tests/milvus_benchmark/suites/011_build_debug.yaml new file mode 100644 index 0000000000..ddc44f89cf --- /dev/null +++ b/tests/milvus_benchmark/suites/011_build_debug.yaml @@ -0,0 +1,92 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_hnsw + cache_config.cpu_cache_capacity: 4GB + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_10m_2000000_128_l2 + index_type: hnsw + index_param: + M: 48 + efConstruction: 500 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_ivf_flat_16384 + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 100 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 6GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_4000000_128_l2 + # index_type: ivf_flat + # index_param: + # nlist: 16384 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8_16384 + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 100 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 6GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_4000000_128_l2 + # index_type: ivf_sq8 + # index_param: + # nlist: 16384 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8h_16384 + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 100 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 6GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_4000000_128_l2 + # index_type: ivf_sq8h + # index_param: + # nlist: 16384 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_pq_16384 + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 100 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 6GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_4000000_128_l2 + # index_type: ivf_pq + # index_param: + # nlist: 16384 + # m: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_compact_performance.yaml b/tests/milvus_benchmark/suites/011_compact_performance.yaml new file mode 100644 index 0000000000..8708505f39 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_compact_performance.yaml @@ -0,0 +1,15 @@ +compact_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_1m_128_128_l2_delete + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_1m_50000_128_l2_2 diff --git a/tests/milvus_benchmark/suites/011_cpu_accuracy.yaml b/tests/milvus_benchmark/suites/011_cpu_accuracy.yaml new file mode 100644 index 0000000000..365434d012 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_accuracy.yaml @@ -0,0 +1,55 @@ +accuracy: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8 + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_10m_100000_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_10m_2000000_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_hnsw + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_10m_2000000_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + ef: [64, 100, 200, 500, 700] diff --git a/tests/milvus_benchmark/suites/011_cpu_accuracy_ann.yaml b/tests/milvus_benchmark/suites/011_cpu_accuracy_ann.yaml new file mode 100644 index 0000000000..a21859e098 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_accuracy_ann.yaml @@ -0,0 +1,271 @@ +ann_accuracy: + collections: + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [500000] + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512, 16384] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [500000] + index_types: ['ivf_flat', 'ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [500000] + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [32] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [500000] + index_types: ['annoy'] + index_params: + n_trees: [8, 32] + top_ks: [10] + nqs: [10000] + search_params: + search_k: [50, 100, 500, 1000] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [500000] + index_types: ['hnsw'] + index_params: + M: [16] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + segment_row_limits: [500000] + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512, 16384] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + segment_row_limits: [500000] + index_types: ['ivf_flat', 'ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + segment_row_limits: [500000] + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [20] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + segment_row_limits: [500000] + index_types: ['annoy'] + index_params: + n_trees: [8, 32] + top_ks: [10] + nqs: [10000] + search_params: + search_k: [50, 100, 500, 1000] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + segment_row_limits: [500000] + index_types: ['hnsw'] + index_params: + M: [36] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [10, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + segment_row_limits: [500000] + index_types: ['nsg'] + index_params: + search_length: 45 + out_degree: 50 + candidate_pool_size: 300 + knng: 100 + top_ks: [10] + nqs: [10000] + search_params: + search_length: [50] diff --git a/tests/milvus_benchmark/suites/011_cpu_accuracy_ann_debug.yaml b/tests/milvus_benchmark/suites/011_cpu_accuracy_ann_debug.yaml new file mode 100644 index 0000000000..ecc6875db8 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_accuracy_ann_debug.yaml @@ -0,0 +1,73 @@ +ann_accuracy: + collections: + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + server: + cpus: 50.0 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [500000] + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512] + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [500000] + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512] +# - +# server: +# cache_config.cpu_cache_capacity: 16GB +# engine_config.use_blas_threshold: 1100 +# engine_config.gpu_search_threshold: 1 +# gpu_resource_config.enable: false +# gpu_resource_config.cache_capacity: 4GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# source_file: /test/milvus/ann_hdf5/kosarak-27983-jaccard.hdf5 +# collection_name: kosarak_27984_jaccard +# segment_row_limits: [100000] +# index_types: ['bin_flat', 'bin_ivf_flat'] +# index_params: +# nlist: [2048] +# top_ks: [10] +# nqs: [10000] +# search_params: +# nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_cpu_accuracy_rhnsw.yaml b/tests/milvus_benchmark/suites/011_cpu_accuracy_rhnsw.yaml new file mode 100644 index 0000000000..68b477a81d --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_accuracy_rhnsw.yaml @@ -0,0 +1,36 @@ +accuracy: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_rhnsw_pq + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_10m_2000000_128_l2 + top_ks: [32] + nqs: [1000] + search_params: + ef: [32, 64, 100] +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_rhnsw_sq +# cache_config.cpu_cache_capacity: 8GB +# engine_config.use_blas_threshold: 1100 +# engine_config.gpu_search_threshold: 1 +# gpu_resource_config.enable: false +# gpu_resource_config.cache_capacity: 4GB +# gpu_resource_config.search_resources: +# - gpu0 +# gpu_resource_config.build_index_resources: +# - gpu0 +# collection_name: sift_50m_4000000_128_l2 +# top_ks: [64] +# nqs: [1000] +# search_params: +# ef: [32, 64, 100, 200, 500] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_cpu_build.yaml b/tests/milvus_benchmark/suites/011_cpu_build.yaml new file mode 100644 index 0000000000..17376451d7 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_build.yaml @@ -0,0 +1,40 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/random_1m_1024_512_l2_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: random_1m_1024_512_l2 + index_type: ivf_flat + index_param: + nlist: 16384 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_1m_128_128_l2_pq + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1m_128_128_l2 + index_type: ivf_pq + index_param: + nlist: 8092 + m: 32 diff --git a/tests/milvus_benchmark/suites/011_cpu_build_binary.yaml b/tests/milvus_benchmark/suites/011_cpu_build_binary.yaml new file mode 100644 index 0000000000..5a749cc67d --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_build_binary.yaml @@ -0,0 +1,11 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/binary_50m_4000000_512_jaccard_ivf + cache_config.cpu_cache_capacity: 8GB + gpu_resource_config.enable: false + collection_name: binary_50m_4000000_512_jaccard + index_type: bin_ivf_flat + index_param: + nlist: 2048 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_cpu_build_hnsw.yaml b/tests/milvus_benchmark/suites/011_cpu_build_hnsw.yaml new file mode 100644 index 0000000000..81d91df920 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_build_hnsw.yaml @@ -0,0 +1,12 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_ip_hnsw + cache_config.cpu_cache_capacity: 8GB + gpu_resource_config.enable: false + collection_name: sift_10m_2000000_128_ip + index_type: hnsw + index_param: + M: 48 + efConstruction: 500 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_cpu_build_rhnsw.yaml b/tests/milvus_benchmark/suites/011_cpu_build_rhnsw.yaml new file mode 100644 index 0000000000..2a9dde4f10 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_build_rhnsw.yaml @@ -0,0 +1,23 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_rhnsw_pq + cache_config.cpu_cache_capacity: 8GB + gpu_resource_config.enable: false + collection_name: sift_10m_2000000_128_l2 + index_type: rhnsw_pq + index_param: + M: 48 + efConstruction: 500 + PQM: 16 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_rhnsw_sq + cache_config.cpu_cache_capacity: 8GB + gpu_resource_config.enable: false + collection_name: sift_50m_4000000_128_l2 + index_type: rhnsw_sq + index_param: + M: 48 + efConstruction: 500 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_cpu_search.yaml b/tests/milvus_benchmark/suites/011_cpu_search.yaml new file mode 100644 index 0000000000..93e015f0a8 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_search.yaml @@ -0,0 +1,255 @@ +search_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 150GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_1b_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2 + cache_config.cpu_cache_capacity: 64GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2 + cache_config.cpu_cache_capacity: 64GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 64GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_pq + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1000] + search_params: + - + nprobe: 8 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_nsg + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + search_length: 50 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_annoy + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + search_k: 100 + - + search_k: 500 + - + search_k: 1000 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/binary_50m_4000000_512_jaccard + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: binary_50m_4000000_512_jaccard + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/binary_50m_4000000_512_jaccard_ivf + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: binary_50m_4000000_512_jaccard + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 diff --git a/tests/milvus_benchmark/suites/011_cpu_search_binary.yaml b/tests/milvus_benchmark/suites/011_cpu_search_binary.yaml new file mode 100644 index 0000000000..e22788b64d --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_search_binary.yaml @@ -0,0 +1,49 @@ +search_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/binary_50m_4000000_512_jaccard + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: binary_50m_4000000_512_jaccard + run_count: 2 + top_ks: [10, 1, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/binary_50m_4000000_512_jaccard_ivf + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: binary_50m_4000000_512_jaccard + run_count: 2 + top_ks: [10, 1, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 diff --git a/tests/milvus_benchmark/suites/011_cpu_search_sift10m.yaml b/tests/milvus_benchmark/suites/011_cpu_search_sift10m.yaml new file mode 100644 index 0000000000..2dd2622851 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_search_sift10m.yaml @@ -0,0 +1,122 @@ +search_performance: + collections: +# - +# milvus: +# db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2 +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_2000000_128_l2 +# run_count: 2 +# top_ks: [1, 1000] +# nqs: [1, 100, 1200] +# search_params: +# - +# nprobe: 8 + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + server: + cpus: 24.0 + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1200] + search_params: + - + nprobe: 8 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_ivf_flat_16384 +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_2000000_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000, 1200] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 +# +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_sq8_16384 +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_2000000_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000, 1200] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 +# +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_pq_16384 +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_2000000_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000, 1200] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_cpu_search_sift10m_100k.yaml b/tests/milvus_benchmark/suites/011_cpu_search_sift10m_100k.yaml new file mode 100644 index 0000000000..f3710a8476 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_search_sift10m_100k.yaml @@ -0,0 +1,97 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1200] + search_params: + - + nprobe: 8 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_pq + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_cpu_search_sift10m_filter.yaml b/tests/milvus_benchmark/suites/011_cpu_search_sift10m_filter.yaml new file mode 100644 index 0000000000..1e645d9523 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_search_sift10m_filter.yaml @@ -0,0 +1,97 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2/ + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1200] + search_params: + - + nprobe: 8 + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_pq + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_cpu_search_sift10m_hnsw.yaml b/tests/milvus_benchmark/suites/011_cpu_search_sift10m_hnsw.yaml new file mode 100644 index 0000000000..5f1b68cb77 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_search_sift10m_hnsw.yaml @@ -0,0 +1,40 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_hnsw + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [100] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + ef: 100 + - + ef: 200 + - + ef: 500 + - + ef: 1000 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_ip_hnsw + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + wal_enable: true + collection_name: sift_10m_2000000_128_ip + run_count: 2 + top_ks: [100] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + ef: 100 + - + ef: 200 + - + ef: 500 + - + ef: 1000 diff --git a/tests/milvus_benchmark/suites/011_cpu_search_sift10m_ivf.yaml b/tests/milvus_benchmark/suites/011_cpu_search_sift10m_ivf.yaml new file mode 100644 index 0000000000..cc8f5ef19c --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_search_sift10m_ivf.yaml @@ -0,0 +1,32 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_ivf_flat_16384 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_ip_ivf_flat_16384 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + wal_enable: true + collection_name: sift_10m_2000000_128_ip + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 diff --git a/tests/milvus_benchmark/suites/011_cpu_search_sift1b.yaml b/tests/milvus_benchmark/suites/011_cpu_search_sift1b.yaml new file mode 100644 index 0000000000..4feeca0f4a --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_search_sift1b.yaml @@ -0,0 +1,26 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 150GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_1b_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_cpu_search_sift50m.yaml b/tests/milvus_benchmark/suites/011_cpu_search_sift50m.yaml new file mode 100644 index 0000000000..5fd7dd5fb9 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_cpu_search_sift50m.yaml @@ -0,0 +1,98 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1200] + search_params: + - + nprobe: 8 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_ivf_flat_16384 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8_16384 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_pq_16384 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_delete_performance.yaml b/tests/milvus_benchmark/suites/011_delete_performance.yaml new file mode 100644 index 0000000000..af902eae92 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_delete_performance.yaml @@ -0,0 +1,17 @@ +delete_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_1m_128_128_l2_delete + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_1m_50000_128_l2_2 + ni_per: 50000 + auto_flush: false diff --git a/tests/milvus_benchmark/suites/011_get_id.yaml b/tests/milvus_benchmark/suites/011_get_id.yaml new file mode 100644 index 0000000000..258ba42c3b --- /dev/null +++ b/tests/milvus_benchmark/suites/011_get_id.yaml @@ -0,0 +1,17 @@ +get_ids_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_10m_1024_128_l2 + ids_length_per_segment: [1, 10, 100] diff --git a/tests/milvus_benchmark/suites/011_gpu_accuracy.yaml b/tests/milvus_benchmark/suites/011_gpu_accuracy.yaml new file mode 100644 index 0000000000..7046a0d275 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_accuracy.yaml @@ -0,0 +1,61 @@ +accuracy: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_10m_2000000_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_10m_2000000_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_ip + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_10m_2000000_128_ip + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1] diff --git a/tests/milvus_benchmark/suites/011_gpu_accuracy_ann.yaml b/tests/milvus_benchmark/suites/011_gpu_accuracy_ann.yaml new file mode 100644 index 0000000000..6aecc767aa --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_accuracy_ann.yaml @@ -0,0 +1,172 @@ +ann_accuracy: + collections: + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [100000] + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [100000] + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [32] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [100000] + index_types: ['hnsw'] + index_params: + M: [16] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + segment_row_limits: [100000] + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + segment_row_limits: [100000] + index_types: ['hnsw'] + index_params: + M: [36] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [10, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/kosarak-27983-jaccard.hdf5 + collection_name: kosarak_27984_jaccard + segment_row_limits: [100000] + index_types: ['bin_flat', 'bin_ivf_flat'] + index_params: + nlist: [2048] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-256-hamming.hdf5 + collection_name: sift_256_hamming + segment_row_limits: [100000] + index_types: ['bin_flat', 'bin_ivf_flat'] + index_params: + nlist: [2048] + top_ks: [100] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tests/milvus_benchmark/suites/011_gpu_accuracy_ann_debug.yaml b/tests/milvus_benchmark/suites/011_gpu_accuracy_ann_debug.yaml new file mode 100644 index 0000000000..ec260cd159 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_accuracy_ann_debug.yaml @@ -0,0 +1,25 @@ +ann_accuracy: + collections: + - + server: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [100000] + index_types: ['ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512] diff --git a/tests/milvus_benchmark/suites/011_gpu_accuracy_debug.yaml b/tests/milvus_benchmark/suites/011_gpu_accuracy_debug.yaml new file mode 100644 index 0000000000..4819f26d90 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_accuracy_debug.yaml @@ -0,0 +1,23 @@ +accuracy: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + - gpu2 + - gpu3 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_4000000_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [32, 64, 128] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_gpu_build.yaml b/tests/milvus_benchmark/suites/011_gpu_build.yaml new file mode 100644 index 0000000000..488dc16b67 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_build.yaml @@ -0,0 +1,21 @@ +build_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8_4096 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_10m_100000_128_l2 + index_type: ivf_sq8 + index_param: + nlist: 4096 + diff --git a/tests/milvus_benchmark/suites/011_gpu_build_debug.yaml b/tests/milvus_benchmark/suites/011_gpu_build_debug.yaml new file mode 100644 index 0000000000..8e534adc92 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_build_debug.yaml @@ -0,0 +1,151 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8_4096 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu2 + collection_name: sift_10m_100000_128_l2 + index_type: ivf_sq8 + index_param: + nlist: 4096 + + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_ivf + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_10m_100000_128_l2 + # index_type: ivf_flat + # index_param: + # nlist: 1024 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8h + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_10m_100000_128_l2 + # index_type: ivf_sq8h + # index_param: + # nlist: 1024 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8 + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_10m_100000_128_l2 + # index_type: ivf_sq8 + # index_param: + # nlist: 1024 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_pq + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_10m_100000_128_l2 + # index_type: ivf_pq + # index_param: + # nlist: 1024 + # m: 32 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_hnsw + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_10m_100000_128_l2 + # index_type: hnsw + # index_param: + # M: 48 + # efConstruction: 500 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_annoy +# cache_config.cpu_cache_capacity: 8GB +# engine_config.use_blas_threshold: 1100 +# engine_config.gpu_search_threshold: 1 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 4GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# collection_name: sift_10m_100000_128_l2 +# index_type: annoy +# index_param: +# n_trees: 8 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_nsg +# cache_config.cpu_cache_capacity: 8GB +# engine_config.use_blas_threshold: 1100 +# engine_config.gpu_search_threshold: 1 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 4GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# collection_name: sift_10m_100000_128_l2 +# index_type: nsg +# index_param: +# search_length: 50 +# out_degree: 40 +# candidate_pool_size: 100 +# knng: 50 diff --git a/tests/milvus_benchmark/suites/011_gpu_build_sift10m.yaml b/tests/milvus_benchmark/suites/011_gpu_build_sift10m.yaml new file mode 100644 index 0000000000..4a782b3c1f --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_build_sift10m.yaml @@ -0,0 +1,148 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + - gpu2 + - gpu3 + collection_name: sift_10m_2000000_128_l2 + index_type: ivf_flat + index_param: + nlist: 1024 + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + - gpu2 + - gpu3 + collection_name: sift_10m_2000000_128_l2 + index_type: ivf_sq8 + index_param: + nlist: 1024 + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_sq8h + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + - gpu2 + - gpu3 + collection_name: sift_10m_2000000_128_l2 + index_type: ivf_sq8h + index_param: + nlist: 1024 + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_pq + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + - gpu2 + - gpu3 + collection_name: sift_10m_2000000_128_l2 + index_type: ivf_pq + index_param: + nlist: 1024 + m: 32 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_hnsw +# cache_config.cpu_cache_capacity: 8GB +# engine_config.use_blas_threshold: 1100 +# engine_config.gpu_search_threshold: 1 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 4GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# - gpu2 +# - gpu3 +# collection_name: sift_10m_100000_128_l2 +# index_type: hnsw +# index_param: +# M: 48 +# efConstruction: 500 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_annoy +# cache_config.cpu_cache_capacity: 8GB +# engine_config.use_blas_threshold: 1100 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 4GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# - gpu2 +# - gpu3 +# collection_name: sift_10m_100000_128_l2 +# index_type: annoy +# index_param: +# n_trees: 8 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_nsg +# cache_config.cpu_cache_capacity: 8GB +# engine_config.use_blas_threshold: 1100 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 4GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# - gpu2 +# - gpu3 +# collection_name: sift_10m_100000_128_l2 +# index_type: nsg +# index_param: +# search_length: 50 +# out_degree: 40 +# candidate_pool_size: 100 +# knng: 50 + diff --git a/tests/milvus_benchmark/suites/011_gpu_build_sift1b.yaml b/tests/milvus_benchmark/suites/011_gpu_build_sift1b.yaml new file mode 100644 index 0000000000..a40e5766b2 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_build_sift1b.yaml @@ -0,0 +1,42 @@ +build_performance: + collections: +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_sq8h +# cache_config.cpu_cache_capacity: 8GB +# engine_config.use_blas_threshold: 1100 +# engine_config.gpu_search_threshold: 1 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 4GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# - gpu2 +# - gpu3 +# collection_name: sift_1b_4000000_128_l2 +# index_type: ivf_sq8h +# index_param: +# nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + - gpu2 + - gpu3 + collection_name: sift_1b_4000000_128_l2 + index_type: ivf_sq8 + index_param: + nlist: 16384 diff --git a/tests/milvus_benchmark/suites/011_gpu_build_sift50m.yaml b/tests/milvus_benchmark/suites/011_gpu_build_sift50m.yaml new file mode 100644 index 0000000000..a9fc44b582 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_build_sift50m.yaml @@ -0,0 +1,75 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_pq + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_4000000_128_l2 + index_type: ivf_pq + index_param: + nlist: 16384 + m: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_4000000_128_l2 + index_type: ivf_flat + index_param: + nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_4000000_128_l2 + index_type: ivf_sq8 + index_param: + nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8h + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_4000000_128_l2 + index_type: ivf_sq8h + index_param: + nlist: 16384 diff --git a/tests/milvus_benchmark/suites/011_gpu_search.yaml b/tests/milvus_benchmark/suites/011_gpu_search.yaml new file mode 100644 index 0000000000..03971028ee --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_search.yaml @@ -0,0 +1,225 @@ +search_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8h + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_pq + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_hnsw + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [100] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + ef: 100 + - + ef: 200 + - + ef: 500 + - + ef: 1000 + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_annoy + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + search_k: 100 + - + search_k: 500 + - + search_k: 1000 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_nsg + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + search_length: 50 +# - +# milvus: +# db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_sq8h +# cache_config.cpu_cache_capacity: 150GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 200 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_1b_4000000_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000, 1200] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 diff --git a/tests/milvus_benchmark/suites/011_gpu_search_debug.yaml b/tests/milvus_benchmark/suites/011_gpu_search_debug.yaml new file mode 100644 index 0000000000..4c5a7dc7cc --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_search_debug.yaml @@ -0,0 +1,73 @@ +search_performance: + collections: +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_pq +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_2000000_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000, 1200] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_sq8h +# cache_config.cpu_cache_capacity: 150GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 200 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_1b_4000000_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000, 1200] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 150GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_1b_4000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_gpu_search_sift10m.yaml b/tests/milvus_benchmark/suites/011_gpu_search_sift10m.yaml new file mode 100644 index 0000000000..bd49cddbd9 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_search_sift10m.yaml @@ -0,0 +1,145 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1200] + search_params: + - + nprobe: 8 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_sq8h + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_pq + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_pq_16384 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_gpu_search_sift10m_100k.yaml b/tests/milvus_benchmark/suites/011_gpu_search_sift10m_100k.yaml new file mode 100644 index 0000000000..e76d235406 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_search_sift10m_100k.yaml @@ -0,0 +1,121 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1200] + search_params: + - + nprobe: 8 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000, 8192] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8h + # cache_config.cpu_cache_capacity: 32GB + # engine_config.use_blas_threshold: 0 + # engine_config.gpu_search_threshold: 100 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 6GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # wal_enable: true + # collection_name: sift_10m_100000_128_l2 + # run_count: 2 + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 1000, 1200] + # search_params: + # - + # nprobe: 8 + # - + # nprobe: 32 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_pq + # cache_config.cpu_cache_capacity: 32GB + # engine_config.use_blas_threshold: 0 + # engine_config.gpu_search_threshold: 100 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 6GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # wal_enable: true + # collection_name: sift_10m_100000_128_l2 + # run_count: 2 + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 1000, 1200] + # search_params: + # - + # nprobe: 8 + # - + # nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_gpu_search_sift10m_filter.yaml b/tests/milvus_benchmark/suites/011_gpu_search_sift10m_filter.yaml new file mode 100644 index 0000000000..b287c65e42 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_search_sift10m_filter.yaml @@ -0,0 +1,122 @@ +search_performance: + collections: +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2/ +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_2000000_128_l2 +# run_count: 2 +# top_ks: [1, 1000] +# nqs: [1, 100, 1200] +# search_params: +# - +# nprobe: 8 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_ivf_flat +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_2000000_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000, 1200] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 + +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_sq8 +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_2000000_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000, 1200] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_sq8h + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + filters: + - + term: "{'term': {'float': {'values': [float(i) for i in range(collection_size // 1000)]}}}" + search_params: + - + nprobe: 32 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_pq +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_2000000_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000, 1200] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_gpu_search_sift10m_ivf.yaml b/tests/milvus_benchmark/suites/011_gpu_search_sift10m_ivf.yaml new file mode 100644 index 0000000000..6b73f5df2a --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_search_sift10m_ivf.yaml @@ -0,0 +1,50 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_ivf_flat_16384 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_ip_ivf_flat_16384 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_ip + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 diff --git a/tests/milvus_benchmark/suites/011_gpu_search_sift50m.yaml b/tests/milvus_benchmark/suites/011_gpu_search_sift50m.yaml new file mode 100644 index 0000000000..7aae432ca2 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_search_sift50m.yaml @@ -0,0 +1,121 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1200] + search_params: + - + nprobe: 8 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2_sq8h + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2_pq + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_gpu_sift50m_ivf.yaml b/tests/milvus_benchmark/suites/011_gpu_sift50m_ivf.yaml new file mode 100644 index 0000000000..8bf700c870 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_sift50m_ivf.yaml @@ -0,0 +1,26 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_gpu_stability.yaml b/tests/milvus_benchmark/suites/011_gpu_stability.yaml new file mode 100644 index 0000000000..9e7ec4f006 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_gpu_stability.yaml @@ -0,0 +1,39 @@ +stability: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_1024_128_l2_sq8_stability + cache_config.cpu_cache_capacity: 64GB + cache_config.cache_insert_data: true + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 50 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1m_50000_128_l2_2 + during_time: 5 + operations: + insert: + weight: 4 + xb: 100 + delete: + weight: 4 + xb: 100 + flush: + weight: 1 + # async: true + compact: + weight: 1 + # # async: true + query: + weight: 2 + # async: true + top_ks: 1-100 + nqs: 1-100 + search_params: + nprobe: 1-100 diff --git a/tests/milvus_benchmark/suites/011_insert_data.yaml b/tests/milvus_benchmark/suites/011_insert_data.yaml new file mode 100644 index 0000000000..6f12acae20 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_insert_data.yaml @@ -0,0 +1,55 @@ +insert_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2 + cache_config.cpu_cache_capacity: 4GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + other_fields: int,float + ni_per: 50000 + build_index: false + index_type: ivf_sq8 + index_param: + nlist: 1024 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2 + # cache_config.cpu_cache_capacity: 4GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # wal_enable: true + # collection_name: sift_50m_100000_128_l2 + # ni_per: 50000 + # build_index: false + # index_type: ivf_sq8 + # index_param: + # nlist: 1024 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_1b_524288_128_l2_debug +# collection_name: sift_1b_524288_128_l2 +# ni_per: 100000 +# # flush: no +# build_index: false +# index_type: ivf_sq8 +# index_param: +# nlist: 4096 diff --git a/tests/milvus_benchmark/suites/011_insert_debug.yaml b/tests/milvus_benchmark/suites/011_insert_debug.yaml new file mode 100644 index 0000000000..b41252ba68 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_insert_debug.yaml @@ -0,0 +1,89 @@ +insert_performance: + collections: + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/binary_50m_4000000_512_jaccard + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # collection_name: binary_50m_4000000_512_jaccard + # ni_per: 100000 + # build_index: false + # index_type: bin_flat + # index_param: + # nlist: 2048 + + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_ip + # cache_config.cpu_cache_capacity: 4GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # wal_enable: true + # collection_name: sift_10m_2000000_128_ip + # ni_per: 100000 + # build_index: false + # index_type: ivf_sq8 + # index_param: + # nlist: 1024 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2 + # cache_config.cpu_cache_capacity: 4GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # wal_enable: true + # collection_name: sift_50m_4000000_128_l2 + # ni_per: 50000 + # build_index: false + # index_type: ivf_sq8 + # index_param: + # nlist: 1024 + + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_debug4 + # cache_config.cpu_cache_capacity: 32GB + # wal_enable: false + # collection_name: sift_1b_4000000_128_l2 + # ni_per: 100000 + # # flush: no + # build_index: false + # index_type: ivf_sq8 + # index_param: + # nlist: 16384 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_debug5 + #cache_config.cpu_cache_capacity: 32GB + wal_enable: true + collection_name: sift_1b_4000000_128_l2 + ni_per: 100000 + # flush: no + build_index: false + index_type: ivf_sq8 + index_param: + nlist: 16384 diff --git a/tests/milvus_benchmark/suites/011_insert_performance.yaml b/tests/milvus_benchmark/suites/011_insert_performance.yaml new file mode 100644 index 0000000000..af9b5a53d6 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_insert_performance.yaml @@ -0,0 +1,95 @@ +insert_performance: + collections: + - + server: + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_5m_100000_128_l2 + ni_per: 10000 + flush: no + build_index: false + index_type: ivf_sq8 + index_param: + nlist: 1024 + - + server: + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_5m_100000_128_l2 + ni_per: 50000 + flush: no + build_index: false + index_type: ivf_sq8 + index_param: + nlist: 1024 + + - + server: + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_5m_100000_128_l2 + ni_per: 100000 + flush: no + build_index: false + index_type: ivf_sq8 + index_param: + nlist: 1024 + + - + server: + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_5m_100000_128_l2 + ni_per: 200000 + flush: no + build_index: false + index_type: ivf_sq8 + index_param: + nlist: 1024 + + - + server: + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_5m_100000_128_l2 + ni_per: 500000 + flush: no + build_index: false + index_type: ivf_sq8 + index_param: + nlist: 1024 diff --git a/tests/milvus_benchmark/suites/011_locust_search.yaml b/tests/milvus_benchmark/suites/011_locust_search.yaml new file mode 100644 index 0000000000..9cb96dbf27 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_locust_search.yaml @@ -0,0 +1,43 @@ +locust_search_performance: + collections: + - + milvus: + cache_config.cpu_cache_capacity: 8GB + cache_config.insert_buffer_size: 2GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_1m_2000000_128_l2_2 + ni_per: 50000 + build_index: true + index_type: ivf_sq8 + index_param: + nlist: 16384 + task: + connection_num: 1 + clients_num: 100 + hatch_rate: 2 + during_time: 5 + types: + - + type: query + weight: 1 + params: + top_k: 10 + nq: 1 + # filters: + # - + # range: + # int64: + # LT: 0 + # GT: 1000000 + search_param: + nprobe: 16 diff --git a/tests/milvus_benchmark/suites/011_search_dsl.yaml b/tests/milvus_benchmark/suites/011_search_dsl.yaml new file mode 100644 index 0000000000..62091cd178 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_search_dsl.yaml @@ -0,0 +1,76 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2/ + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1200] + filters: + - + term: "{'term': {'float': {'values': [float(i) for i in range(collection_size // 2)]}}}" + - + range: "{'range': {'int64': {'LT': 0, 'GT':collection_size // 2}}}" + - + range: "{'range': {'int64': {'LT': 0, 'GT':collection_size}}}" + - + term: "{'term': {'float': {'values': [float(i) for i in range(collection_size)]}}}" + - + range: "{'range': {'int64': {'LT': 0, 'GT':collection_size // 100000}}}" + - + range: "{'range': {'int64': {'LT': collection_size // 2, 'GT': collection_size}}}" + term: "{'term': {'float': {'values': [float(i) for i in range(collection_size // 2)]}}}" + search_params: + - + nprobe: 8 + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + filters: + - + term: "{'term': {'float': {'values': [float(i) for i in range(collection_size // 2)]}}}" + - + range: "{'range': {'int64': {'LT': 0, 'GT':collection_size // 2}}}" + - + range: "{'range': {'int64': {'LT': 0, 'GT':collection_size}}}" + - + term: "{'term': {'float': {'values': [float(i) for i in range(collection_size)]}}}" + - + range: "{'range': {'int64': {'LT': 0, 'GT':collection_size // 100000}}}" + - + range: "{'range': {'int64': {'LT': collection_size // 2, 'GT': collection_size}}}" + term: "{'term': {'float': {'values': [float(i) for i in range(collection_size // 2)]}}}" + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_search_stability.yaml b/tests/milvus_benchmark/suites/011_search_stability.yaml new file mode 100644 index 0000000000..837b017628 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_search_stability.yaml @@ -0,0 +1,20 @@ +search_stability: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8_16384_stability + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_50m_4000000_128_l2 + during_time: 180 + top_ks: 1-200 + nqs: 1-200 + search_params: + nprobe: 1-100 diff --git a/tests/milvus_benchmark/suites/011_search_threshold.yaml b/tests/milvus_benchmark/suites/011_search_threshold.yaml new file mode 100644 index 0000000000..d512f25ab5 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_search_threshold.yaml @@ -0,0 +1,50 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 1300 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/011_sift50m_acc.yaml b/tests/milvus_benchmark/suites/011_sift50m_acc.yaml new file mode 100644 index 0000000000..5fa6c50003 --- /dev/null +++ b/tests/milvus_benchmark/suites/011_sift50m_acc.yaml @@ -0,0 +1,19 @@ +accuracy: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_50m_4000000_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/add_flush_performance.yaml b/tests/milvus_benchmark/suites/add_flush_performance.yaml new file mode 100644 index 0000000000..d8c1308732 --- /dev/null +++ b/tests/milvus_benchmark/suites/add_flush_performance.yaml @@ -0,0 +1,20 @@ +insert_flush_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_080/sift_2m_128_128_l2_flush + cache_config.cpu_cache_capacity: 8 + cache_config.insert_buffer_size: 2 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + db_config.auto_flush_interval: 300 + collection_name: sift_2m_128_128_l2 + ni_per: 100000 diff --git a/tests/milvus_benchmark/suites/ann_debug.yaml b/tests/milvus_benchmark/suites/ann_debug.yaml new file mode 100644 index 0000000000..7ed7c948c3 --- /dev/null +++ b/tests/milvus_benchmark/suites/ann_debug.yaml @@ -0,0 +1,26 @@ +ann_accuracy: + collections: + - + server: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_file_sizes: [1024] + index_types: ['ivf_sq8'] + index_params: + nlist: [1024] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 8] + diff --git a/tests/milvus_benchmark/suites/clean.yaml b/tests/milvus_benchmark/suites/clean.yaml new file mode 100644 index 0000000000..18d722ed4b --- /dev/null +++ b/tests/milvus_benchmark/suites/clean.yaml @@ -0,0 +1,24 @@ +search_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_7/not_exist + cache_config.cpu_cache_capacity: 64 + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_2048_128_l2 + run_count: 2 + top_ks: [1] + nqs: [1] + search_params: + - + nprobe: 8 diff --git a/tests/milvus_benchmark/suites/cpu_accuracy.yaml b/tests/milvus_benchmark/suites/cpu_accuracy.yaml new file mode 100644 index 0000000000..d780178089 --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_accuracy.yaml @@ -0,0 +1,61 @@ +accuracy: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_50m_2048_128_ip_sq8_wal + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_2048_128_ip + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_50m_2048_128_l2_hnsw_wal + cache_config.cpu_cache_capacity: 64 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_2048_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + ef: [64, 100, 200, 500, 700] + + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_1b_2048_128_l2_sq8_wal + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml b/tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml index 21cc7995f5..546ed12e59 100644 --- a/tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml +++ b/tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml @@ -15,8 +15,32 @@ ann_accuracy: - gpu1 source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 collection_name: sift_128_euclidean - index_file_sizes: [1024] - index_types: ['flat', 'ivf_flat', 'ivf_sq8'] + segment_row_limits: [100000] + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512, 16384] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [100000] + index_types: ['ivf_flat', 'ivf_sq8'] index_params: nlist: [16384] top_ks: [10] @@ -39,7 +63,7 @@ ann_accuracy: - gpu1 source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 collection_name: sift_128_euclidean - index_file_sizes: [1024] + segment_row_limits: [100000] index_types: ['ivf_pq'] index_params: nlist: [16384] @@ -64,7 +88,7 @@ ann_accuracy: - gpu1 source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 collection_name: sift_128_euclidean - index_file_sizes: [1024] + segment_row_limits: [100000] index_types: ['annoy'] index_params: n_trees: [8, 32] @@ -88,7 +112,7 @@ ann_accuracy: - gpu1 source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 collection_name: sift_128_euclidean - index_file_sizes: [256] + segment_row_limits: [100000] index_types: ['hnsw'] index_params: M: [16] @@ -113,8 +137,32 @@ ann_accuracy: - gpu1 source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 collection_name: glove_200_angular - index_file_sizes: [1024] - index_types: ['flat', 'ivf_flat', 'ivf_sq8'] + segment_row_limits: [100000] + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512, 16384] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + segment_row_limits: [100000] + index_types: ['ivf_flat', 'ivf_sq8'] index_params: nlist: [16384] top_ks: [10] @@ -137,7 +185,7 @@ ann_accuracy: - gpu1 source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 collection_name: glove_200_angular - index_file_sizes: [1024] + segment_row_limits: [100000] index_types: ['ivf_pq'] index_params: nlist: [16384] @@ -162,7 +210,7 @@ ann_accuracy: - gpu1 source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 collection_name: glove_200_angular - index_file_sizes: [256] + segment_row_limits: [100000] index_types: ['hnsw'] index_params: M: [36] diff --git a/tests/milvus_benchmark/suites/cpu_accuracy_ann_debug.yaml b/tests/milvus_benchmark/suites/cpu_accuracy_ann_debug.yaml new file mode 100644 index 0000000000..019a05c06d --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_accuracy_ann_debug.yaml @@ -0,0 +1,25 @@ +ann_accuracy: + collections: + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_file_sizes: [1024] + index_types: ['ivf_sq8', 'ivf_sq8h'] + index_params: + nlist: 2048 + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [100] + nqs: [1000] diff --git a/tests/milvus_benchmark/suites/cpu_accuracy_ann_hnsw.yaml b/tests/milvus_benchmark/suites/cpu_accuracy_ann_hnsw.yaml new file mode 100644 index 0000000000..8f649e136d --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_accuracy_ann_hnsw.yaml @@ -0,0 +1,43 @@ +ann_accuracy: + collections: + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_file_sizes: [256] + index_types: ['hnsw'] + nlists: [16384] + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_file_sizes: [256] + index_types: ['hnsw'] + nlists: [16384] + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] diff --git a/tests/milvus_benchmark/suites/cpu_accuracy_ann_pq.yaml b/tests/milvus_benchmark/suites/cpu_accuracy_ann_pq.yaml new file mode 100644 index 0000000000..1e087d3fbb --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_accuracy_ann_pq.yaml @@ -0,0 +1,26 @@ +ann_accuracy: + collections: + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_file_sizes: [1024] + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [20] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tests/milvus_benchmark/suites/cpu_accuracy_nsg.yaml b/tests/milvus_benchmark/suites/cpu_accuracy_nsg.yaml new file mode 100644 index 0000000000..c2e6194e10 --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_accuracy_nsg.yaml @@ -0,0 +1,21 @@ +accuracy: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_10m_1024_128_l2_nsg_wal + cache_config.cpu_cache_capacity: 50 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_10m_1024_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + search_length: [50] diff --git a/tests/milvus_benchmark/suites/cpu_build_performance.yaml b/tests/milvus_benchmark/suites/cpu_build_performance.yaml new file mode 100644 index 0000000000..660aff14f5 --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_build_performance.yaml @@ -0,0 +1,19 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_070/sift_50m_1024_128_l2_sq8h_wal + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + index_type: ivf_sq8h + nlist: 16384 diff --git a/tests/milvus_benchmark/suites/cpu_search_binary.yaml b/tests/milvus_benchmark/suites/cpu_search_binary.yaml new file mode 100644 index 0000000000..37aa2d0a35 --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_search_binary.yaml @@ -0,0 +1,67 @@ +search_performance: + collections: + # - + # server: + # db_config.primary_path: /test/milvus/db_data_7/sub_50m_512_512_sub_wal + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # collection_name: sub_50m_512_512_sub + # run_count: 2 + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # search_params: + # - + # nprobe: 8 + # - + # nprobe: 32 + + # - + # server: + # db_config.primary_path: /test/milvus/db_data_7/super_50m_512_512_super_wal + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # collection_name: super_50m_512_512_super + # run_count: 2 + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # search_params: + # - + # nprobe: 8 + # - + # nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_7/jaccard_50m_512_512_jaccard_wal + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: jaccard_50m_512_512_jaccard + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 diff --git a/tests/milvus_benchmark/suites/cpu_search_performance_jaccard.yaml b/tests/milvus_benchmark/suites/cpu_search_performance_jaccard.yaml new file mode 100644 index 0000000000..0266cababe --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_search_performance_jaccard.yaml @@ -0,0 +1,20 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_070/jaccard_50m_512_512_jaccard_wal + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: jaccard_50m_512_512_jaccard + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] diff --git a/tests/milvus_benchmark/suites/cpu_search_performance_sift1b.yaml b/tests/milvus_benchmark/suites/cpu_search_performance_sift1b.yaml new file mode 100644 index 0000000000..f4a48f9a15 --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_search_performance_sift1b.yaml @@ -0,0 +1,22 @@ +search_performance: + collections: + + # sift_1b + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_1b_2048_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] diff --git a/tests/milvus_benchmark/suites/cpu_search_performance_sift1m.yaml b/tests/milvus_benchmark/suites/cpu_search_performance_sift1m.yaml new file mode 100644 index 0000000000..dfc8e53637 --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_search_performance_sift1m.yaml @@ -0,0 +1,20 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_1m_1024_128_l2 + run_count: 1 + search_params: + nprobes: [8] + top_ks: [1, 16, 64] + nqs: [1, 10, 100] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/cpu_search_performance_sift50m.yaml b/tests/milvus_benchmark/suites/cpu_search_performance_sift50m.yaml new file mode 100644 index 0000000000..b74bb9e56e --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_search_performance_sift50m.yaml @@ -0,0 +1,20 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] diff --git a/tests/milvus_benchmark/suites/cpu_stability_sift50m.yaml b/tests/milvus_benchmark/suites/cpu_stability_sift50m.yaml new file mode 100644 index 0000000000..61b203173e --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_stability_sift50m.yaml @@ -0,0 +1,27 @@ +stability: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192_stability + cache_config.cpu_cache_capacity: 64 + cache_config.cache_insert_data: true + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + during_time: 480 + search_params: + nprobes: 1-200 + top_ks: 1-200 + nqs: 1-200 + # length of insert vectors + insert_xb: 100000 + # insert after search 4 times + insert_interval: 4 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/debug.yaml b/tests/milvus_benchmark/suites/debug.yaml new file mode 100644 index 0000000000..07c0297ef7 --- /dev/null +++ b/tests/milvus_benchmark/suites/debug.yaml @@ -0,0 +1,90 @@ +search_performance: + collections: +# - +# server: +# db_config.primary_path: /test/milvus/db_data_8/sift_1b_2048_128_l2_sq8_wal +# cache_config.cpu_cache_capacity: 150GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 200 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_1b_2048_128_l2 +# run_count: 4 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000] +# search_params: +# - +# nprobe: 8 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_7/sift_1b_2048_128_l2_sq8h_wal +# cache_config.cpu_cache_capacity: 150 +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 200 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6 +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_1b_2048_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 200, 500, 1000] +# search_params: +# - +# nprobe: 8 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_7/sift_50m_2048_128_l2_ivf_wal +# cache_config.cpu_cache_capacity: 64 +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 200 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6 +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_50m_2048_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 200, 500, 1000] +# search_params: +# - +# nprobe: 8 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_7/sift_50m_2048_128_l2_sq8h_wal +# cache_config.cpu_cache_capacity: 64 +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 200 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6 +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_50m_2048_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 200, 500, 1000] +# search_params: +# - +# nprobe: 8 diff --git a/tests/milvus_benchmark/suites/debug_build.yaml b/tests/milvus_benchmark/suites/debug_build.yaml new file mode 100644 index 0000000000..09566b5c9c --- /dev/null +++ b/tests/milvus_benchmark/suites/debug_build.yaml @@ -0,0 +1,23 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_10m_1024_128_l2_nsg_wal + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_10m_1024_128_l2 + index_type: nsg + index_param: + search_length: 50 + out_degree: 40 + candidate_pool_size: 100 + knng: 50 diff --git a/tests/milvus_benchmark/suites/debug_gpu_search.yaml b/tests/milvus_benchmark/suites/debug_gpu_search.yaml new file mode 100644 index 0000000000..eefb968e99 --- /dev/null +++ b/tests/milvus_benchmark/suites/debug_gpu_search.yaml @@ -0,0 +1,26 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_50m_2048_128_l2_ivf_wal + cache_config.cpu_cache_capacity: 64 + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_2048_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 diff --git a/tests/milvus_benchmark/suites/file_size_debug.yaml b/tests/milvus_benchmark/suites/file_size_debug.yaml new file mode 100644 index 0000000000..fe96684ab0 --- /dev/null +++ b/tests/milvus_benchmark/suites/file_size_debug.yaml @@ -0,0 +1,44 @@ +insert_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu_crud/file_size/sift_20m_2048_128_l2 + cache_config.cpu_cache_capacity: 32 + cache_config.insert_buffer_size: 2 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: false + collection_name: sift_20m_2048_128_l2 + ni_per: 100000 + build_index: true + index_type: ivf_sq8 + nlist: 16384 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu_crud/file_size/sift_20m_1024_128_l2 + # cache_config.cpu_cache_capacity: 32 + # cache_config.insert_buffer_size: 2 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1100 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # wal_enable: false + # collection_name: sift_20m_1024_128_l2 + # ni_per: 100000 + # build_index: true + # index_type: ivf_sq8 + # nlist: 16384 diff --git a/tests/milvus_benchmark/suites/gpu_accuracy.yaml b/tests/milvus_benchmark/suites/gpu_accuracy.yaml new file mode 100644 index 0000000000..2f5a086210 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_accuracy.yaml @@ -0,0 +1,41 @@ +accuracy: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_50m_2048_128_l2_sq8_wal + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_2048_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_1b_2048_128_l2_sq8h_wal + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml index 1b7bcf248a..900abf73ed 100644 --- a/tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml +++ b/tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml @@ -2,11 +2,11 @@ ann_accuracy: collections: - server: - cache_config.cpu_cache_capacity: 16 + cache_config.cpu_cache_capacity: 16GB engine_config.use_blas_threshold: 1100 engine_config.gpu_search_threshold: 1 gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 + gpu_resource_config.cache_capacity: 4GB gpu_resource_config.search_resources: - gpu0 - gpu1 @@ -22,15 +22,15 @@ ann_accuracy: top_ks: [10] nqs: [10000] search_params: - nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 16384] - server: - cache_config.cpu_cache_capacity: 16 + cache_config.cpu_cache_capacity: 16GB engine_config.use_blas_threshold: 1100 engine_config.gpu_search_threshold: 1 gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 + gpu_resource_config.cache_capacity: 4GB gpu_resource_config.search_resources: - gpu0 - gpu1 @@ -51,11 +51,11 @@ ann_accuracy: - server: - cache_config.cpu_cache_capacity: 16 + cache_config.cpu_cache_capacity: 16GB engine_config.use_blas_threshold: 1100 engine_config.gpu_search_threshold: 1 gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 + gpu_resource_config.cache_capacity: 4GB gpu_resource_config.search_resources: - gpu0 - gpu1 @@ -96,15 +96,15 @@ ann_accuracy: top_ks: [10] nqs: [10000] search_params: - nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 16384] - server: - cache_config.cpu_cache_capacity: 16 + cache_config.cpu_cache_capacity: 16GB engine_config.use_blas_threshold: 1100 engine_config.gpu_search_threshold: 1 gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 + gpu_resource_config.cache_capacity: 4GB gpu_resource_config.search_resources: - gpu0 - gpu1 @@ -125,11 +125,11 @@ ann_accuracy: - server: - cache_config.cpu_cache_capacity: 16 + cache_config.cpu_cache_capacity: 16GB engine_config.use_blas_threshold: 1100 engine_config.gpu_search_threshold: 1 gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 + gpu_resource_config.cache_capacity: 4GB gpu_resource_config.search_resources: - gpu0 - gpu1 @@ -149,11 +149,11 @@ ann_accuracy: - server: - cache_config.cpu_cache_capacity: 16 + cache_config.cpu_cache_capacity: 16GB engine_config.use_blas_threshold: 1100 engine_config.gpu_search_threshold: 1 gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 + gpu_resource_config.cache_capacity: 4GB gpu_resource_config.search_resources: - gpu0 - gpu1 diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_ann_debug.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_ann_debug.yaml new file mode 100644 index 0000000000..7adb14cf5d --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_accuracy_ann_debug.yaml @@ -0,0 +1,25 @@ +ann_accuracy: + collections: + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + segment_row_limits: [100000] + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512, 16384] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_debug.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_debug.yaml new file mode 100644 index 0000000000..4c4b75f725 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_accuracy_debug.yaml @@ -0,0 +1,40 @@ +accuracy: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_1b_2048_128_l2_sq8h_wal + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_1b_2048_128_l2_sq8h_wal + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_sift1b.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_sift1b.yaml new file mode 100644 index 0000000000..dbfe2abe8c --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_accuracy_sift1b.yaml @@ -0,0 +1,59 @@ +accuracy: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_sift1m.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_sift1m.yaml new file mode 100644 index 0000000000..0955622270 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_accuracy_sift1m.yaml @@ -0,0 +1,21 @@ +accuracy: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_1m_256_128_l2_sq8 + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1m_256_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tests/milvus_benchmark/suites/gpu_build_performance_hamming50m.yaml b/tests/milvus_benchmark/suites/gpu_build_performance_hamming50m.yaml new file mode 100644 index 0000000000..538b6e31fb --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_build_performance_hamming50m.yaml @@ -0,0 +1,36 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_ivf + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: hamming_50m_128_512_hamming + index_type: ivf_flat + nlist: 2048 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_flat + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: hamming_50m_128_512_hamming + index_type: flat + nlist: 2048 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_build_performance_jaccard50m.yaml b/tests/milvus_benchmark/suites/gpu_build_performance_jaccard50m.yaml new file mode 100644 index 0000000000..fdf9cccd26 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_build_performance_jaccard50m.yaml @@ -0,0 +1,20 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/jaccard_50m_512_512_jaccard_ivf_wal_debug + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: jaccard_50m_512_512_jaccard + index_type: ivf_flat + index_param: + nlist: 2048 diff --git a/tests/milvus_benchmark/suites/gpu_build_pq.yaml b/tests/milvus_benchmark/suites/gpu_build_pq.yaml new file mode 100644 index 0000000000..3ddd890684 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_build_pq.yaml @@ -0,0 +1,47 @@ +search_performance: + collections: + # - + # server: + # db_config.primary_path: /test/milvus/db_data_8/sift_10m_1024_128_l2_pq_64 + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 6 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_10m_1024_128_l2 + # index_type: ivf_pq + # index_param: + # nlist: 10000 + # m: 64 + + - + server: + db_config.primary_path: /test/milvus/db_data_8/sift_10m_1024_128_l2_pq + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_1024_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 + diff --git a/tests/milvus_benchmark/suites/gpu_build_sift1b_sq8h.yaml b/tests/milvus_benchmark/suites/gpu_build_sift1b_sq8h.yaml new file mode 100644 index 0000000000..a4d823cafb --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_build_sift1b_sq8h.yaml @@ -0,0 +1,20 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_1b_2048_128_l2_sq8h_wal + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + index_type: ivf_sq8h + index_param: + nlist: 16384 diff --git a/tests/milvus_benchmark/suites/gpu_search_performance.yaml b/tests/milvus_benchmark/suites/gpu_search_performance.yaml new file mode 100644 index 0000000000..78f6334a7f --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance.yaml @@ -0,0 +1,247 @@ +search_performance: + collections: + # sift_50m + - + server: + db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_50m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_50m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_50m_1024_128_l2_sq8h + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_50m_1024_128_l2_nsg + # cache_config.cpu_cache_capacity: 50 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + # random_50m + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu_crud/random_50m_1024_512_ip_ivf + # cache_config.cpu_cache_capacity: 110 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: random_50m_1024_512_ip + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8 + # cache_config.cpu_cache_capacity: 30 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: random_50m_1024_512_ip + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8h + # cache_config.cpu_cache_capacity: 30 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: random_50m_1024_512_ip + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_nsg + # cache_config.cpu_cache_capacity: 200 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 6 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: random_50m_1024_512_ip + # run_count: 2 + # search_params: + # nprobes: [8] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + # sift_1b + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_1b_1024_128_l2_sq8 + # cache_config.cpu_cache_capacity: 150 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_1b_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_1b_2048_128_l2_sq8h + # cache_config.cpu_cache_capacity: 150 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_1b_2048_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq + # cache_config.cpu_cache_capacity: 150 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_1b_2048_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_hamming50m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_hamming50m.yaml new file mode 100644 index 0000000000..4e55559f84 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance_hamming50m.yaml @@ -0,0 +1,22 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: hamming_50m_128_512_hamming + run_count: 1 + search_params: + nprobes: [8, 32] + top_ks: [1, 16, 64, 128, 256, 512, 1000] + nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_jaccard50m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_jaccard50m.yaml new file mode 100644 index 0000000000..c9a1ed99be --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance_jaccard50m.yaml @@ -0,0 +1,22 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: jaccard_50m_128_512_jaccard + run_count: 1 + search_params: + nprobes: [8, 32] + top_ks: [1, 16, 64, 128, 256, 512, 1000] + nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_sift1b.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_sift1b.yaml new file mode 100644 index 0000000000..2e317d9861 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance_sift1b.yaml @@ -0,0 +1,62 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_sift1m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_sift1m.yaml new file mode 100644 index 0000000000..f9349f1e03 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance_sift1m.yaml @@ -0,0 +1,22 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1m_1024_128_l2 + run_count: 1 + search_params: + nprobes: [8] + top_ks: [1, 16, 64] + nqs: [1, 10, 100] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_sift50m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_sift50m.yaml new file mode 100644 index 0000000000..4f49bf4937 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance_sift50m.yaml @@ -0,0 +1,146 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 + # cache_config.cpu_cache_capacity: 16 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h + # cache_config.cpu_cache_capacity: 16 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + # git issue num: #626 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg + # cache_config.cpu_cache_capacity: 50 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192 + # cache_config.cpu_cache_capacity: 16 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 + # cache_config.cpu_cache_capacity: 16 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_stability.yaml b/tests/milvus_benchmark/suites/gpu_search_stability.yaml new file mode 100644 index 0000000000..a41ea817cd --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_stability.yaml @@ -0,0 +1,23 @@ +search_stability: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_50m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 50 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + - gpu2 + - gpu3 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_50m_1024_128_l2 + during_time: 240 + search_params: + nprobes: 1-200 + top_ks: 1-200 + nqs: 1-200 diff --git a/tests/milvus_benchmark/suites/gpu_stability_sift50m.yaml b/tests/milvus_benchmark/suites/gpu_stability_sift50m.yaml new file mode 100644 index 0000000000..56faf7b205 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_stability_sift50m.yaml @@ -0,0 +1,27 @@ +stability: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h_stability + cache_config.cpu_cache_capacity: 64 + cache_config.cache_insert_data: true + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + during_time: 480 + search_params: + nprobes: 1-200 + top_ks: 1-200 + nqs: 1-200 + # length of insert vectors + insert_xb: 100000 + # insert after search 4 times + insert_interval: 4 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert_binary.yaml b/tests/milvus_benchmark/suites/insert_binary.yaml new file mode 100644 index 0000000000..79fa2b3567 --- /dev/null +++ b/tests/milvus_benchmark/suites/insert_binary.yaml @@ -0,0 +1,39 @@ +insert_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sub_50m_512_512_sub_wal + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sub_50m_512_512_sub + ni_per: 100000 + build_index: false + index_type: flat + index_param: + nlist: 2048 + + - + server: + db_config.primary_path: /test/milvus/db_data_7/super_50m_512_512_super_wal + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: super_50m_512_512_super + ni_per: 100000 + build_index: false + index_type: flat + index_param: + nlist: 2048 diff --git a/tests/milvus_benchmark/suites/insert_performance_deep1b.yaml b/tests/milvus_benchmark/suites/insert_performance_deep1b.yaml new file mode 100644 index 0000000000..27dc83c63e --- /dev/null +++ b/tests/milvus_benchmark/suites/insert_performance_deep1b.yaml @@ -0,0 +1,87 @@ +insert_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_ivf + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: ivf_flat + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_sq8 + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: ivf_sq8 + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_sq8h + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: ivf_sq8h + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_pq + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: ivf_pq + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_nsg + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: nsg + # nlist: 16384 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/locust_insert.yaml b/tests/milvus_benchmark/suites/locust_insert.yaml new file mode 100644 index 0000000000..cdbdeaca20 --- /dev/null +++ b/tests/milvus_benchmark/suites/locust_insert.yaml @@ -0,0 +1,23 @@ +locust_insert_performance: + collections: + - + server: + cache_config.cpu_cache_capacity: 8GB + cache_config.insert_buffer_size: 2GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + task: + type: insert + connection_num: 1 + clients_num: 10 + hatch_rate: 5 + during_time: 2m diff --git a/tests/milvus_benchmark/suites/locust_mix.yaml b/tests/milvus_benchmark/suites/locust_mix.yaml new file mode 100644 index 0000000000..088256f582 --- /dev/null +++ b/tests/milvus_benchmark/suites/locust_mix.yaml @@ -0,0 +1,53 @@ +locust_mix_performance: + collections: + - + milvus: + cache_config.insert_buffer_size: 2GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + gpu0 + gpu_resource_config.build_index_resources: + gpu0 + wal_enable: true + collection_name: sift_1m_50000_128_l2 + other_fields: int + ni_per: 50000 + build_index: true + index_type: ivf_sq8 + index_param: + nlist: 16384 + tasks: + types: + - type: flush + weight: 1 + - + type: query + weight: 30 + params: + top_k: 10 + nq: 100 + filters: + - range: + int64: + LT: 0 + GT: 1000000 + search_param: + nprobe: 16 + - + type: insert + weight: 10 + params: + nb: 1 + - + type: delete + weight: 5 + - + type: compact + weight: 1 + connection_num: 1 + clients_num: 1000 + hatch_rate: 100 + during_time: 5h diff --git a/tests/milvus_benchmark/suites/locust_search.yaml b/tests/milvus_benchmark/suites/locust_search.yaml new file mode 100644 index 0000000000..a8196c2fe1 --- /dev/null +++ b/tests/milvus_benchmark/suites/locust_search.yaml @@ -0,0 +1,34 @@ +locust_search_performance_2: + collections: + - + server: + cache_config.cpu_cache_capacity: 8GB + cache_config.insert_buffer_size: 2GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_1m_2000000_128_l2_2 + ni_per: 50000 + build_index: false + index_type: ivf_sq8 + index_param: + nlist: 16384 + task: + type: query + connection_num: 1 + clients_num: 100 + spawn_rate: 2 + during_time: 60 + params: + top_k: 10 + nq: 1 + search_param: + nprobe: 16 diff --git a/tests/milvus_benchmark/suites/loop_stability.yaml b/tests/milvus_benchmark/suites/loop_stability.yaml new file mode 100644 index 0000000000..a304695feb --- /dev/null +++ b/tests/milvus_benchmark/suites/loop_stability.yaml @@ -0,0 +1,17 @@ +loop_stability: + collections: + - + server: + suffix_path: true + db_config.primary_path: /test/milvus/db_data_11/loop_stability + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 10 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 2GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + pull_interval: 20 + collection_num: 4 diff --git a/tests/milvus_benchmark/suites/metric.yaml b/tests/milvus_benchmark/suites/metric.yaml new file mode 100644 index 0000000000..263176b6e8 --- /dev/null +++ b/tests/milvus_benchmark/suites/metric.yaml @@ -0,0 +1,47 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_50m_2048_128_l2_sq8_wal + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 0 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_2048_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 64 + +# - +# server: +# db_config.primary_path: /test/milvus/db_data_7/sift_50m_2048_128_ip_sq8_wal +# cache_config.cpu_cache_capacity: 32 +# engine_config.use_blas_threshold: 1100 +# engine_config.gpu_search_threshold: 200 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6 +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_50m_2048_128_ip +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 200, 500, 1000] +# search_params: +# - +# nprobe: 64 diff --git a/tests/milvus_benchmark/suites/poseidon.yaml b/tests/milvus_benchmark/suites/poseidon.yaml new file mode 100644 index 0000000000..6cd9472b9b --- /dev/null +++ b/tests/milvus_benchmark/suites/poseidon.yaml @@ -0,0 +1,26 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_8/sift_50m_2048_128_l2_ivf + cache_config.cpu_cache_capacity: 64GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_2048_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 diff --git a/tests/milvus_benchmark/suites/pq.yaml b/tests/milvus_benchmark/suites/pq.yaml new file mode 100644 index 0000000000..6e69156ac0 --- /dev/null +++ b/tests/milvus_benchmark/suites/pq.yaml @@ -0,0 +1,27 @@ +ann_accuracy: + collections: + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_file_sizes: [1024] + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [20] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tests/milvus_benchmark/suites/qps.yaml b/tests/milvus_benchmark/suites/qps.yaml new file mode 100644 index 0000000000..6e0b94c527 --- /dev/null +++ b/tests/milvus_benchmark/suites/qps.yaml @@ -0,0 +1,27 @@ +search_performance_concurrents: + collections: + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + concurrents: [2000] + use_single_connection: false + index_file_size: [1024] + index_types: ['ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [1] + search_params: + nprobe: [16] diff --git a/tests/milvus_benchmark/suites/search_debug.yaml b/tests/milvus_benchmark/suites/search_debug.yaml new file mode 100644 index 0000000000..84f04600fb --- /dev/null +++ b/tests/milvus_benchmark/suites/search_debug.yaml @@ -0,0 +1,92 @@ +search_performance: + collections: +# - +# server: +# db_config.primary_path: /test/milvus/db_data_8/sift_1b_2048_128_l2_sq8 +# cache_config.cpu_cache_capacity: 150 +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 200 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6 +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_1b_2048_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 200, 500, 1000] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_8/sift_1b_2048_128_l2_sq8 +# cache_config.cpu_cache_capacity: 150 +# engine_config.use_blas_threshold: 1100 +# engine_config.gpu_search_threshold: 200 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6 +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_1b_2048_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 200, 500, 1000] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_rhnsw_pq + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + wal_enable: true + collection_name: sift_10m_2000000_128_l2 + run_count: 2 + top_ks: [100] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + ef: 100 + - + ef: 200 + - + ef: 500 + - + ef: 1000 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_rhnsw_sq + cache_config.cpu_cache_capacity: 64GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + wal_enable: true + collection_name: sift_50m_4000000_128_l2 + run_count: 2 + top_ks: [100] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + ef: 100 + - + ef: 200 + - + ef: 500 + - + ef: 1000 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/shards_ann_debug.yaml b/tests/milvus_benchmark/suites/shards_ann_debug.yaml new file mode 100644 index 0000000000..ba5db54eb4 --- /dev/null +++ b/tests/milvus_benchmark/suites/shards_ann_debug.yaml @@ -0,0 +1,25 @@ +ann_accuracy: + collections: + - + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_file_sizes: [1024] + index_types: ['flat', 'ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 32, 512] + + - + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_file_sizes: [1024] + index_types: ['flat', 'ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 32, 512] diff --git a/tests/milvus_benchmark/suites/shards_insert_performance.yaml b/tests/milvus_benchmark/suites/shards_insert_performance.yaml new file mode 100644 index 0000000000..eada67a3b4 --- /dev/null +++ b/tests/milvus_benchmark/suites/shards_insert_performance.yaml @@ -0,0 +1,17 @@ +insert_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_8/shards_sift_1m_128_128_l2_insert + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_1m_128_128_l2 + ni_per: 10000 + build_index: false + index_type: flat diff --git a/tests/milvus_benchmark/suites/shards_insert_performance_sift1m.yaml b/tests/milvus_benchmark/suites/shards_insert_performance_sift1m.yaml new file mode 100644 index 0000000000..dbc2929fda --- /dev/null +++ b/tests/milvus_benchmark/suites/shards_insert_performance_sift1m.yaml @@ -0,0 +1,19 @@ +insert_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_010/shards_sift_1m_128_128_l2_insert + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_1m_1024_128_l2 + ni_per: 10000 + build_index: true + index_type: ivf_sq8 + index_param: + nlist: 16384 diff --git a/tests/milvus_benchmark/suites/shards_loop_stability.yaml b/tests/milvus_benchmark/suites/shards_loop_stability.yaml new file mode 100644 index 0000000000..4494b8f4b3 --- /dev/null +++ b/tests/milvus_benchmark/suites/shards_loop_stability.yaml @@ -0,0 +1,16 @@ +loop_stability: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_8/shards_loop_stability + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 10 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 2GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + pull_interval: 2 + collection_num: 2 diff --git a/tests/milvus_benchmark/suites/shards_search_performance_sift1m.yaml b/tests/milvus_benchmark/suites/shards_search_performance_sift1m.yaml new file mode 100644 index 0000000000..71ed51ce72 --- /dev/null +++ b/tests/milvus_benchmark/suites/shards_search_performance_sift1m.yaml @@ -0,0 +1,12 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_010/shards_sift_1m_128_128_l2_insert + wal_enable: true + collection_name: sift_1m_1024_128_l2 + run_count: 2 + top_ks: [1, 10, 100] + nqs: [1, 10, 100] + search_params: + - nprobe: 8 diff --git a/tests/milvus_benchmark/task/task.py b/tests/milvus_benchmark/task/task.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/milvus_benchmark/test_loop.py b/tests/milvus_benchmark/test_loop.py new file mode 100644 index 0000000000..224c796a08 --- /dev/null +++ b/tests/milvus_benchmark/test_loop.py @@ -0,0 +1,52 @@ +import time +import random +import logging +from client import MilvusClient +import utils + + +if __name__ == "__main__": + milvus_instance = MilvusClient() + milvus_instance.clean_db() + p_num = 1 + dimension = 128 + insert_xb = 100000 + index_types = ['flat'] + index_param = {"nlist": 2048} + collection_names = [] + milvus_instances_map = {} + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + + for i in range(collection_num): + name = utils.get_unique_name(prefix="collection_") + print(name) + collection_names.append(name) + metric_type = "ip" + # metric_type = random.choice(["l2", "ip"]) + index_file_size = random.randint(10, 100) + milvus_instance.create_collection(name, dimension, index_file_size, metric_type) + milvus_instance = MilvusClient(collection_name=name) + index_type = random.choice(index_types) + milvus_instance.create_index(index_type, index_param=index_param) + insert_vectors = utils.normalize(metric_type, insert_vectors) + milvus_instance.insert(insert_vectors) + milvus_instance.flush() + milvus_instances_map.update({name: milvus_instance}) + print(milvus_instance.describe_index(), milvus_instance.describe(), milvus_instance.count()) + + # tasks = ["insert_rand", "delete_rand", "query_rand", "flush", "compact"] + tasks = ["insert_rand", "query_rand", "flush"] + i = 1 + while True: + print("Loop time: %d" % i) + start_time = time.time() + while time.time() - start_time < pull_interval_seconds: + # choose collection + tmp_collection_name = random.choice(collection_names) + # choose task from task + task_name = random.choice(tasks) + # print(tmp_collection_name, task_name) + func = getattr(milvus_instances_map[tmp_collection_name], task_name) + func() + print("Restart") + i = i + 1 diff --git a/tests/milvus_benchmark/utils.py b/tests/milvus_benchmark/utils.py index 3c6a43c991..987fc46626 100644 --- a/tests/milvus_benchmark/utils.py +++ b/tests/milvus_benchmark/utils.py @@ -1,8 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function - -__true_print = print # noqa - import os import sys import pdb @@ -15,22 +11,165 @@ import logging import string import random # import multiprocessing -# import numpy +import numpy as np # import psutil +import sklearn.preprocessing import h5py # import docker from yaml import full_load, dump +import yaml import tableprint as tp from pprint import pprint - +from milvus import DataType logger = logging.getLogger("milvus_benchmark.utils") -MULTI_DB_SLAVE_PATH = "/opt/milvus/data2;/opt/milvus/data3" -REGISTRY_URL = "registry.zilliz.com/milvus/engine" +DEFAULT_F_FIELD_NAME = 'float_vector' +DEFAULT_B_FIELD_NAME = 'binary_vector' +DEFAULT_INT_FIELD_NAME = 'int64' +DEFAULT_FLOAT_FIELD_NAME = 'float' -def get_unique_name(): - return "benchmark-test-"+"".join(random.choice(string.ascii_letters + string.digits) for _ in range(8)).lower() +METRIC_MAP = { + "l2": "L2", + "ip": "IP", + "jaccard": "JACCARD", + "hamming": "HAMMING", + "sub": "SUBSTRUCTURE", + "super": "SUPERSTRUCTURE" +} + + +def metric_type_trans(metric_type): + if metric_type in METRIC_MAP.keys(): + return METRIC_MAP[metric_type] + else: + raise Exception("metric_type: %s not in METRIC_MAP" % metric_type) + + +def timestr_to_int(time_str): + time_int = 0 + if isinstance(time_str, int) or time_str.isdigit(): + time_int = int(time_str) + elif time_str.endswith("s"): + time_int = int(time_str.split("s")[0]) + elif time_str.endswith("m"): + time_int = int(time_str.split("m")[0]) * 60 + elif time_str.endswith("h"): + time_int = int(time_str.split("h")[0]) * 60 * 60 + else: + raise Exception("%s not support" % time_str) + return time_int + + +class literal_str(str): pass + + +def change_style(style, representer): + def new_representer(dumper, data): + scalar = representer(dumper, data) + scalar.style = style + return scalar + + return new_representer + + +from yaml.representer import SafeRepresenter + +# represent_str does handle some corner cases, so use that +# instead of calling represent_scalar directly +represent_literal_str = change_style('|', SafeRepresenter.represent_str) + +yaml.add_representer(literal_str, represent_literal_str) + + +def retry(times): + """ + This decorator prints the execution time for the decorated function. + """ + def wrapper(func): + def newfn(*args, **kwargs): + attempt = 0 + while attempt < times: + try: + logger.info(attempt) + return func(*args, **kwargs) + except Exception as e: + logger.info(str(e)) + time.sleep(3) + attempt += 1 + return func(*args, **kwargs) + return newfn + return wrapper + + +def timestr_to_int(time_str): + time_int = 0 + if isinstance(time_str, int) or time_str.isdigit(): + time_int = int(time_str) + elif time_str.endswith("s"): + time_int = int(time_str.split("s")[0]) + elif time_str.endswith("m"): + time_int = int(time_str.split("m")[0]) * 60 + elif time_str.endswith("h"): + time_int = int(time_str.split("h")[0]) * 60 * 60 + else: + raise Exception("%s not support" % time_str) + return time_int + + +def get_default_field_name(data_type=DataType.FLOAT_VECTOR): + if data_type == DataType.FLOAT_VECTOR: + field_name = DEFAULT_F_FIELD_NAME + elif data_type == DataType.BINARY_VECTOR: + field_name = DEFAULT_B_FIELD_NAME + elif data_type == DataType.INT64: + field_name = DEFAULT_INT_FIELD_NAME + elif data_type == DataType.FLOAT: + field_name = DEFAULT_FLOAT_FIELD_NAME + else: + logger.error(data_type) + raise Exception("Not supported data type") + return field_name + + +def normalize(metric_type, X): + if metric_type == "ip": + logger.info("Set normalize for metric_type: %s" % metric_type) + X = sklearn.preprocessing.normalize(X, axis=1, norm='l2') + X = X.tolist() + elif metric_type in ["jaccard", "hamming", "sub", "super"]: + tmp = [] + for index, item in enumerate(X): + new_vector = bytes(np.packbits(item, axis=-1).tolist()) + tmp.append(new_vector) + X = tmp + return X + + +def convert_nested(dct): + def insert(dct, lst): + for x in lst[:-2]: + dct[x] = dct = dct.get(x, dict()) + dct.update({lst[-2]: lst[-1]}) + + # empty dict to store the result + + result = dict() + + # create an iterator of lists + # representing nested or hierarchial flow + lsts = ([*k.split("."), v] for k, v in dct.items()) + + # insert each list into the result + for lst in lsts: + insert(result, lst) + return result + + +def get_unique_name(prefix=None): + if prefix is None: + prefix = "milvus-benchmark-test-" + return prefix + "".join(random.choice(string.ascii_letters + string.digits) for _ in range(8)).lower() def get_current_time(): @@ -72,11 +211,11 @@ def modify_config(k, v, type=None, file_path="conf/server_config.yaml", db_slave config_dict['gpu_resource_config']['cache_capacity'] = int(v) elif k.find("index_build_device") != -1: config_dict['gpu_resource_config']['build_index_resources'] = v - elif k.find("search_resources") != -1: + elif k.find("search_resources") != -1: config_dict['resource_config']['resources'] = v - if db_slave: - config_dict['db_config']['db_slave_path'] = MULTI_DB_SLAVE_PATH + # if db_slave: + # config_dict['db_config']['db_slave_path'] = MULTI_DB_SLAVE_PATH with open(file_path, 'w') as f: dump(config_dict, f, default_flow_style=False) f.close() @@ -108,294 +247,8 @@ def update_server_config(file_path, server_config): values_dict['gpu_resource_config']['cache_capacity'] = int(v) elif k.find("build_index_resources") != -1: values_dict['gpu_resource_config']['build_index_resources'] = v - elif k.find("search_resources") != -1: + elif k.find("search_resources") != -1: values_dict['gpu_resource_config']['search_resources'] = v with open(file_path, 'w') as f: dump(values_dict, f, default_flow_style=False) f.close() - - -# update values.yaml -def update_values(file_path, hostname, server_config): - from kubernetes import client, config - client.rest.logger.setLevel(logging.WARNING) - - if not os.path.isfile(file_path): - raise Exception('File: %s not found' % file_path) - #　bak values.yaml - file_name = os.path.basename(file_path) - bak_file_name = file_name+".bak" - file_parent_path = os.path.dirname(file_path) - bak_file_path = file_parent_path+'/'+bak_file_name - if os.path.exists(bak_file_path): - os.system("cp %s %s" % (bak_file_path, file_path)) - else: - os.system("cp %s %s" % (file_path, bak_file_path)) - with open(file_path) as f: - values_dict = full_load(f) - f.close() - - for k, v in server_config.items(): - if k.find("primary_path") != -1: - values_dict["primaryPath"] = v - values_dict['wal']['path'] = v+"/wal" - elif k.find("use_blas_threshold") != -1: - values_dict['useBLASThreshold'] = int(v) - elif k.find("gpu_search_threshold") != -1: - values_dict['gpuSearchThreshold'] = int(v) - elif k.find("cpu_cache_capacity") != -1: - values_dict['cpuCacheCapacity'] = int(v) - elif k.find("cache_insert_data") != -1: - values_dict['cacheInsertData'] = v - elif k.find("insert_buffer_size") != -1: - values_dict['insertBufferSize'] = v - elif k.find("gpu_resource_config.enable") != -1: - values_dict['gpu']['enabled'] = v - elif k.find("gpu_resource_config.cache_capacity") != -1: - values_dict['gpu']['cacheCapacity'] = int(v) - elif k.find("build_index_resources") != -1: - values_dict['gpu']['buildIndexResources'] = v - elif k.find("search_resources") != -1: - values_dict['gpu']['searchResources'] = v - # wal - elif k.find("auto_flush_interval") != -1: - values_dict['autoFlushInterval'] = v - elif k.find("wal_enable") != -1: - values_dict['wal']['enabled'] = v - - # if values_dict['nodeSelector']: - # logger.warning("nodeSelector has been set: %s" % str(values_dict['engine']['nodeSelector'])) - # return - values_dict["wal"]["ignoreErrorLog"] = True - # enable monitor - values_dict["metrics"]["enabled"] = True - values_dict["metrics"]["address"] = "192.168.1.237" - values_dict["metrics"]["port"] = 9091 - # update values.yaml with the given host - values_dict['nodeSelector'] = {'kubernetes.io/hostname': hostname} - # Using sqlite - values_dict["mysql"]["enabled"] = False - - config.load_kube_config() - v1 = client.CoreV1Api() - # node = v1.read_node(hostname) - cpus = v1.read_node(hostname).status.allocatable.get("cpu") - # DEBUG - # set limit/request cpus in resources - values_dict['resources'] = { - "limits": { - "cpu": str(int(cpus))+".0" - }, - "requests": { - "cpu": str(int(cpus)-1)+".0" - } - } - values_dict['tolerations'] = [{ - "key": "worker", - "operator": "Equal", - "value": "performance", - "effect": "NoSchedule" - }] - # add extra volumes - values_dict['extraVolumes'] = [{ - 'name': 'test', - 'flexVolume': { - 'driver': "fstab/cifs", - 'fsType': "cifs", - 'secretRef': { - 'name': "cifs-test-secret" - }, - 'options': { - 'networkPath': "//192.168.1.126/test", - 'mountOptions': "vers=1.0" - } - } - }] - values_dict['extraVolumeMounts'] = [{ - 'name': 'test', - 'mountPath': '/test' - }] - logger.debug(values_dict) - with open(file_path, 'w') as f: - dump(values_dict, f, default_flow_style=False) - f.close() - - -# deploy server -def helm_install_server(helm_path, image_tag, image_type, name, namespace): - from kubernetes import client, config - client.rest.logger.setLevel(logging.WARNING) - - timeout = 300 - install_cmd = "helm install --wait --timeout %ds \ - --set image.repository=%s \ - --set image.tag=%s \ - --set image.pullPolicy=Always \ - --set service.type=ClusterIP \ - -f ci/filebeat/values.yaml \ - --namespace %s \ - %s ." % (timeout, REGISTRY_URL, image_tag, namespace, name) - logger.debug(install_cmd) - if os.system("cd %s && %s" % (helm_path, install_cmd)): - logger.error("Helm install failed") - return None - time.sleep(5) - config.load_kube_config() - v1 = client.CoreV1Api() - host = "%s.%s.svc.cluster.local" % (name, namespace) - logger.debug(host) - pod_name = None - pod_id = None - pods = v1.list_namespaced_pod(namespace) - for i in pods.items: - if i.metadata.name.find(name) != -1: - pod_name = i.metadata.name - pod_ip = i.status.pod_ip - logger.debug(pod_name) - logger.debug(pod_ip) - return pod_name, pod_ip - - -# delete server -def helm_del_server(name, namespace): - # del_cmd = "helm uninstall -n milvus benchmark-test-gzelwvgk" - # os.system(del_cmd) - del_cmd = "helm uninstall -n milvus %s" % name - logger.debug(del_cmd) - if os.system(del_cmd): - logger.error("Helm delete name:%s failed" % name) - return False - return True - - -# def pull_image(image): -# registry = image.split(":")[0] -# image_tag = image.split(":")[1] -# client = docker.APIClient(base_url='unix://var/run/docker.sock') -# logger.info("Start pulling image: %s" % image) -# return client.pull(registry, image_tag) - - -# def run_server(image, mem_limit=None, timeout=30, test_type="local", volume_name=None, db_slave=None): -# import colors - -# client = docker.from_env() -# # if mem_limit is None: -# # mem_limit = psutil.virtual_memory().available -# # logger.info('Memory limit:', mem_limit) -# # cpu_limit = "0-%d" % (multiprocessing.cpu_count() - 1) -# # logger.info('Running on CPUs:', cpu_limit) -# for dir_item in ['logs', 'db']: -# try: -# os.mkdir(os.path.abspath(dir_item)) -# except Exception as e: -# pass - -# if test_type == "local": -# volumes = { -# os.path.abspath('conf'): -# {'bind': '/opt/milvus/conf', 'mode': 'ro'}, -# os.path.abspath('logs'): -# {'bind': '/opt/milvus/logs', 'mode': 'rw'}, -# os.path.abspath('db'): -# {'bind': '/opt/milvus/db', 'mode': 'rw'}, -# } -# elif test_type == "remote": -# if volume_name is None: -# raise Exception("No volume name") -# remote_log_dir = volume_name+'/logs' -# remote_db_dir = volume_name+'/db' - -# for dir_item in [remote_log_dir, remote_db_dir]: -# if not os.path.isdir(dir_item): -# os.makedirs(dir_item, exist_ok=True) -# volumes = { -# os.path.abspath('conf'): -# {'bind': '/opt/milvus/conf', 'mode': 'ro'}, -# remote_log_dir: -# {'bind': '/opt/milvus/logs', 'mode': 'rw'}, -# remote_db_dir: -# {'bind': '/opt/milvus/db', 'mode': 'rw'} -# } -# # add volumes -# if db_slave and isinstance(db_slave, int): -# for i in range(2, db_slave+1): -# remote_db_dir = volume_name+'/data'+str(i) -# if not os.path.isdir(remote_db_dir): -# os.makedirs(remote_db_dir, exist_ok=True) -# volumes[remote_db_dir] = {'bind': '/opt/milvus/data'+str(i), 'mode': 'rw'} - -# container = client.containers.run( -# image, -# volumes=volumes, -# runtime="nvidia", -# ports={'19530/tcp': 19530, '8080/tcp': 8080}, -# # environment=["OMP_NUM_THREADS=48"], -# # cpuset_cpus=cpu_limit, -# # mem_limit=mem_limit, -# # environment=[""], -# detach=True) - -# def stream_logs(): -# for line in container.logs(stream=True): -# logger.info(colors.color(line.decode().rstrip(), fg='blue')) - -# if sys.version_info >= (3, 0): -# t = threading.Thread(target=stream_logs, daemon=True) -# else: -# t = threading.Thread(target=stream_logs) -# t.daemon = True -# t.start() - -# logger.info('Container: %s started' % container) -# return container -# # exit_code = container.wait(timeout=timeout) -# # # Exit if exit code -# # if exit_code == 0: -# # return container -# # elif exit_code is not None: -# # print(colors.color(container.logs().decode(), fg='red')) -# # raise Exception('Child process raised exception %s' % str(exit_code)) - -# def restart_server(container): -# client = docker.APIClient(base_url='unix://var/run/docker.sock') - -# client.restart(container.name) -# logger.info('Container: %s restarted' % container.name) -# return container - - -# def remove_container(container): -# container.remove(force=True) -# logger.info('Container: %s removed' % container) - - -# def remove_all_containers(image): -# client = docker.from_env() -# try: -# for container in client.containers.list(): -# if image in container.image.tags: -# container.stop(timeout=30) -# container.remove(force=True) -# except Exception as e: -# logger.error("Containers removed failed") - - -# def container_exists(image): -# ''' -# Check if container existed with the given image name -# @params: image name -# @return: container if exists -# ''' -# res = False -# client = docker.from_env() -# for container in client.containers.list(): -# if image in container.image.tags: -# # True -# res = container -# return res - - -if __name__ == '__main__': - # print(pull_image('branch-0.3.1-debug')) - stop_server()