[skip ci] update benchamrk version to 0.11.1 (#4236)

Signed-off-by: ThreadDao <zongyufen@foxmail.com>

tests/milvus_benchmark/.gitignore

@@ -6,5 +6,7 @@ logs/
 __pycache__/
 venv
 .idea
+nohup.out
 
-suites/
+*.swp
+*.swo

tests/milvus_benchmark/ci/jenkinsfile/cleanup.groovy

@@ -1,12 +1,12 @@
 try {
-    def result = sh script: "helm status benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
+    def result = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true
     if (!result) {
-        sh "helm del --purge benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}"
+        sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}"
     }
 } catch (exc) {
-    def result = sh script: "helm status benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
+    def result = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true
     if (!result) {
-        sh "helm del --purge benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}"
+        sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}"
     }
     throw exc
 }

tests/milvus_benchmark/ci/jenkinsfile/cleanupShards.groovy

@@ -0,0 +1,13 @@
+try {
+    def result = sh script: "helm status -n milvus ${env.HELM_SHARDS_RELEASE_NAME}", returnStatus: true
+    if (!result) {
+        sh "helm uninstall -n milvus ${env.HELM_SHARDS_RELEASE_NAME}"
+    }
+} catch (exc) {
+    def result = sh script: "helm status -n milvus ${env.HELM_SHARDS_RELEASE_NAME}", returnStatus: true
+    if (!result) {
+        sh "helm uninstall -n milvus ${env.HELM_SHARDS_RELEASE_NAME}"
+    }
+    throw exc
+}
+

tests/milvus_benchmark/ci/jenkinsfile/deploy_shards_test.groovy

@@ -0,0 +1,21 @@
+timeout(time: 12, unit: 'HOURS') {
+    try {
+        dir ("milvus-helm") {
+            // sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+            // sh 'helm repo update'
+            checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]])
+        }
+        dir ("milvus_benchmark") {
+            print "Git clone url: ${TEST_URL}:${TEST_BRANCH}"
+            checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]])
+            print "Install requirements"
+            // sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com"
+            sh "python3 -m pip install -r requirements.txt"
+            sh "python3 -m pip install git+${TEST_LIB_URL}"
+            sh "python3 main.py --image-version=${params.IMAGE_VERSION} --schedule-conf=scheduler/${params.SHARDS_CONFIG_FILE} --deploy-mode=${params.DEPLOY_MODE}"
+        }
+    } catch (exc) {
+        echo 'Deploy SHARDS Test Failed !'
+        throw exc
+    }
+}

tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy

@@ -1,21 +1,19 @@
-timeout(time: 4000, unit: 'MINUTES') {
-    try {
-        dir ("milvus-helm") {
-            // sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
-            // sh 'helm repo update'
-            checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]])
-        }
-        dir ("milvus_benchmark") {
-            print "Git clone url: ${TEST_URL}:${TEST_BRANCH}"
-            checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]])
-            print "Install requirements"
-            // sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com"
-            sh "python3 -m pip install -r requirements.txt"
-            sh "python3 -m pip install git+${TEST_LIB_URL}"
-            sh "python3 main.py --image-version=${params.IMAGE_VERSION} --schedule-conf=scheduler/${params.CONFIG_FILE}"
-        }
-    } catch (exc) {
-        echo 'Deploy Test Failed !'
-        throw exc
+try {
+    dir ("milvus-helm") {
+        // sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+        // sh 'helm repo update'
+        checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]])
     }
+    dir ("milvus_benchmark") {
+        print "Git clone url: ${TEST_URL}:${TEST_BRANCH}"
+        checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]])
+        print "Install requirements"
+        // sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com"
+        sh "python3 -m pip install -r requirements.txt"
+        sh "python3 -m pip install git+${TEST_LIB_URL}"
+        sh "python3 main.py --image-version=${params.IMAGE_VERSION} --schedule-conf=scheduler/${params.CONFIG_FILE} --deploy-mode=${params.DEPLOY_MODE}"
+    }
+} catch (exc) {
+    echo 'Deploy Test Failed !'
+    throw exc
 }

tests/milvus_benchmark/ci/jenkinsfile/publishDailyImages.groovy

@@ -1,4 +1,4 @@
-timeout(time: 15, unit: 'MINUTES') {
+timeout(time: 30, unit: 'MINUTES') {
     def imageName = "milvus/engine:${DOCKER_VERSION}"
     def remoteImageName = "milvusdb/daily-build:${REMOTE_DOCKER_VERSION}"
     def localDockerRegistryImage = "${params.LOCAL_DOKCER_REGISTRY_URL}/${imageName}"

tests/milvus_benchmark/ci/main_jenkinsfile

@@ -6,17 +6,21 @@ pipeline {
     }
 
     parameters{
-        string defaultValue: 'master', description: 'server image version', name: 'IMAGE_VERSION', trim: true
-        string defaultValue: '080_data.json', description: 'test suite config yaml', name: 'CONFIG_FILE', trim: true
+        string defaultValue: '0.11.1', description: 'server image version', name: 'IMAGE_VERSION', trim: true
+        choice choices: ['single', 'shards'], description: 'server deploy mode', name: 'DEPLOY_MODE'
+        string defaultValue: '011_data.json', description: 'test suite config yaml', name: 'CONFIG_FILE', trim: true
+        string defaultValue: 'shards.json', description: 'shards test suite config yaml', name: 'SHARDS_CONFIG_FILE', trim: true
         string defaultValue: '09509e53-9125-4f5d-9ce8-42855987ad67', description: 'git credentials', name: 'GIT_USER', trim: true
     }
 
     environment {
         HELM_URL = "https://github.com/milvus-io/milvus-helm.git"
-        HELM_BRANCH = "master"
+        HELM_BRANCH = "0.11.0"
         TEST_URL = "git@192.168.1.105:Test/milvus_benchmark.git"
-        TEST_BRANCH = "master"
+        TEST_BRANCH = "0.11.1"
         TEST_LIB_URL = "http://192.168.1.105:6060/Test/milvus_metrics.git"
+        HELM_RELEASE_NAME = "milvus-benchmark-test-${env.BUILD_NUMBER}"
+        HELM_SHARDS_RELEASE_NAME = "milvus-shards-benchmark-test-${env.BUILD_NUMBER}"
     }
 
     stages {
@@ -35,7 +39,7 @@ pipeline {
                         spec:
                           containers:
                           - name: milvus-test-env
-                            image: registry.zilliz.com/milvus/milvus-test-env:v0.2
+                            image: registry.zilliz.com/milvus/milvus-test-env:v0.3
                             command:
                             - cat
                             tty: true
@@ -93,11 +97,13 @@ pipeline {
 
                 stage("Deploy Test") {
                     steps {
-                        gitlabCommitStatus(name: 'Deploy Test') {
-                            container('milvus-test-env') {
-                                script {
-                                    print "In Deploy Test Stage"
+                        container('milvus-test-env') {
+                            script {
+                                print "In Deploy Test Stage"
+                                if ("${params.DEPLOY_MODE}" == "single") {
                                     load "${env.WORKSPACE}/ci/jenkinsfile/deploy_test.groovy"
+                                } else {
+                                    load "${env.WORKSPACE}/ci/jenkinsfile/deploy_shards_test.groovy"
                                 }
                             }
                         }
@@ -106,10 +112,12 @@ pipeline {
 
                 stage ("Cleanup Env") {
                     steps {
-                        gitlabCommitStatus(name: 'Cleanup Env') {
-                            container('milvus-test-env') {
-                                script {
+                        container('milvus-test-env') {
+                            script {
+                                if ("${params.DEPLOY_MODE}" == "single") {
                                     load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy"
+                                } else {
+                                    load "${env.WORKSPACE}/ci/jenkinsfile/cleanupShards.groovy"
                                 }
                             }
                         }
@@ -117,13 +125,6 @@ pipeline {
                 }
             }
             post {
-                always {
-                    container('milvus-test-env') {
-                        script {
-                            load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy"
-                        }
-                    }
-                }
                 success {
                     script {
                         echo "Milvus benchmark test success !"

tests/milvus_benchmark/ci/publish_jenkinsfile

@@ -9,6 +9,7 @@ pipeline {
         string defaultValue: 'registry.zilliz.com', description: 'Local Docker registry URL', name: 'LOCAL_DOKCER_REGISTRY_URL', trim: true
         string defaultValue: 'registry-1.docker.io', description: 'Remote Docker registry URL', name: 'REMOTE_DOKCER_REGISTRY_URL', trim: true
         string defaultValue: 'milvus-docker-access-token', description: 'Remote Docker credentials id', name: 'REMOTE_DOCKER_CREDENTIALS_ID', trim: true
+        string(defaultValue: "master", description: 'Milvus server version', name: 'BRANCH')
     }
 
     environment {
@@ -41,9 +42,9 @@ pipeline {
                 stages {
                     stage("Publish Docker Images") {
                         environment {
-                            DOCKER_VERSION = "master-${BINARY_VERSION}-${OS_NAME}-release"
-                            REMOTE_DOCKER_VERSION = "${OS_NAME}-${BINARY_VERSION}-${DAILY_BUILD_VERSION}"
-                            REMOTE_DOCKER_LATEST_VERSION = "${OS_NAME}-${BINARY_VERSION}-latest"
+                            DOCKER_VERSION = "${params.BRANCH}-${BINARY_VERSION}-${OS_NAME}-release"
+                            REMOTE_DOCKER_VERSION = "${params.BRANCH}-${OS_NAME}-${BINARY_VERSION}-${DAILY_BUILD_VERSION}"
+                            REMOTE_DOCKER_LATEST_VERSION = "${params.BRANCH}-${OS_NAME}-${BINARY_VERSION}-latest"
                         }
 
                         agent {

tests/milvus_benchmark/client.py

@@ -5,64 +5,76 @@ import logging
 import json
 import time, datetime
 from multiprocessing import Process
-from milvus import Milvus, IndexType, MetricType
+from milvus import Milvus, DataType
+import numpy as np
+import utils
 
 logger = logging.getLogger("milvus_benchmark.client")
 
 SERVER_HOST_DEFAULT = "127.0.0.1"
-# SERVER_HOST_DEFAULT = "192.168.1.130"
 SERVER_PORT_DEFAULT = 19530
 INDEX_MAP = {
-    "flat": IndexType.FLAT,
-    "ivf_flat": IndexType.IVFLAT,
-    "ivf_sq8": IndexType.IVF_SQ8,
-    "nsg": IndexType.RNSG,
-    "ivf_sq8h": IndexType.IVF_SQ8H,
-    "ivf_pq": IndexType.IVF_PQ,
-    "hnsw": IndexType.HNSW,
-    "annoy": IndexType.ANNOY
+    "flat": "FLAT",
+    "ivf_flat": "IVF_FLAT",
+    "ivf_sq8": "IVF_SQ8",
+    "nsg": "NSG",
+    "ivf_sq8h": "IVF_SQ8_HYBRID",
+    "ivf_pq": "IVF_PQ",
+    "hnsw": "HNSW",
+    "annoy": "ANNOY",
+    "bin_flat": "BIN_FLAT",
+    "bin_ivf_flat": "BIN_IVF_FLAT",
+    "rhnsw_pq": "RHNSW_PQ",
+    "rhnsw_sq": "RHNSW_SQ"
 }
 epsilon = 0.1
 
+
 def time_wrapper(func):
     """
     This decorator prints the execution time for the decorated function.
     """
+
     def wrapper(*args, **kwargs):
         start = time.time()
+        logger.info("Milvus {} start".format(func.__name__))
         result = func(*args, **kwargs)
         end = time.time()
         logger.info("Milvus {} run in {}s".format(func.__name__, round(end - start, 2)))
         return result
+
     return wrapper
 
 
 class MilvusClient(object):
-    def __init__(self, collection_name=None, ip=None, port=None, timeout=60):
+    def __init__(self, collection_name=None, host=None, port=None, timeout=180):
         self._collection_name = collection_name
-        try:
-            i = 1
-            start_time = time.time()
-            if not ip:
+        start_time = time.time()
+        if not host:
+            host = SERVER_HOST_DEFAULT
+        if not port:
+            port = SERVER_PORT_DEFAULT
+        logger.debug(host)
+        logger.debug(port)
+        # retry connect remote server
+        i = 0
+        while time.time() < start_time + timeout:
+            try:
                 self._milvus = Milvus(
-                    host = SERVER_HOST_DEFAULT,
-                    port = SERVER_PORT_DEFAULT)
-            else:
-                # retry connect for remote server
-                while time.time() < start_time + timeout:
-                    try:
-                        self._milvus = Milvus(
-                            host = ip,
-                            port = port)
-                        if self._milvus.server_status():
-                            logger.debug("Try connect times: %d, %s" % (i, round(time.time() - start_time, 2)))
-                            break
-                    except Exception as e:
-                        logger.debug("Milvus connect failed")
-                        i = i + 1
+                    host=host,
+                    port=port,
+                    try_connect=False,
+                    pre_ping=False)
+                break
+            except Exception as e:
+                logger.error(str(e))
+                logger.error("Milvus connect failed: %d times" % i)
+                i = i + 1
+                time.sleep(i)
 
-        except Exception as e:
-            raise e
+        if time.time() > start_time + timeout:
+            raise Exception("Server connect timeout")
+        # self._metric_type = None
 
     def __str__(self):
         return 'Milvus collection %s' % self._collection_name
@@ -70,6 +82,8 @@ class MilvusClient(object):
     def check_status(self, status):
         if not status.OK():
             logger.error(status.message)
+            logger.error(self._milvus.server_status())
+            logger.error(self.count())
             raise Exception("Status not ok")
 
     def check_result_ids(self, result):
@@ -79,126 +93,281 @@ class MilvusClient(object):
                 logger.error(item[0].distance)
                 raise Exception("Distance wrong")
 
-    def create_collection(self, collection_name, dimension, index_file_size, metric_type):
-        if not self._collection_name:
-            self._collection_name = collection_name
-        if metric_type == "l2":
-            metric_type = MetricType.L2
-        elif metric_type == "ip":
-            metric_type = MetricType.IP
-        elif metric_type == "jaccard":
-            metric_type = MetricType.JACCARD
-        elif metric_type == "hamming":
-            metric_type = MetricType.HAMMING
-        elif metric_type == "sub":
-            metric_type = MetricType.SUBSTRUCTURE
-        elif metric_type == "super":
-            metric_type = MetricType.SUPERSTRUCTURE
-        else:
-            logger.error("Not supported metric_type: %s" % metric_type)
-        create_param = {'collection_name': collection_name,
-                 'dimension': dimension,
-                 'index_file_size': index_file_size, 
-                 "metric_type": metric_type}
-        status = self._milvus.create_collection(create_param)
+    # only support the given field name
+    def create_collection(self, dimension, segment_row_limit, data_type=DataType.FLOAT_VECTOR, auto_id=False,
+                          collection_name=None, other_fields=None):
+        self._dimension = dimension
+        if not collection_name:
+            collection_name = self._collection_name
+        vec_field_name = utils.get_default_field_name(data_type)
+        fields = [{"name": vec_field_name, "type": data_type, "params": {"dim": dimension}}]
+        if other_fields:
+            other_fields = other_fields.split(",")
+            if "int" in other_fields:
+                fields.append({"name": utils.DEFAULT_INT_FIELD_NAME, "type": DataType.INT64})
+            if "float" in other_fields:
+                fields.append({"name": utils.DEFAULT_FLOAT_FIELD_NAME, "type": DataType.FLOAT})
+        create_param = {
+            "fields": fields,
+            "segment_row_limit": segment_row_limit,
+            "auto_id": auto_id}
+        try:
+            self._milvus.create_collection(collection_name, create_param)
+            logger.info("Create collection: <%s> successfully" % collection_name)
+        except Exception as e:
+            logger.error(str(e))
+            raise
+
+    def generate_values(self, data_type, vectors, ids):
+        values = None
+        if data_type in [DataType.INT32, DataType.INT64]:
+            values = ids
+        elif data_type in [DataType.FLOAT, DataType.DOUBLE]:
+            values = [(i + 0.0) for i in ids]
+        elif data_type in [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR]:
+            values = vectors
+        return values
+
+    def generate_entities(self, vectors, ids=None, collection_name=None):
+        entities = []
+        if collection_name is None:
+            collection_name = self._collection_name
+        info = self.get_info(collection_name)
+        for field in info["fields"]:
+            field_type = field["type"]
+            entities.append(
+                {"name": field["name"], "type": field_type, "values": self.generate_values(field_type, vectors, ids)})
+        return entities
+
+    @time_wrapper
+    def insert(self, entities, ids=None, collection_name=None):
+        tmp_collection_name = self._collection_name if collection_name is None else collection_name
+        insert_ids = self._milvus.insert(tmp_collection_name, entities, ids=ids)
+        return insert_ids
+
+    def insert_rand(self):
+        insert_xb = random.randint(1, 100)
+        X = [[random.random() for _ in range(self._dimension)] for _ in range(insert_xb)]
+        ids = []
+        for i in range(insert_xb):
+            ids.append(random.randint(1, 10000000))
+        entities = self.generate_entities(X, ids)
+        count_before = self.count()
+        res_ids = self.insert(entities, ids=ids)
+        assert res_ids == ids
+        self.flush()
+        assert count_before + insert_xb == self.count()
+
+    def get_rand_ids(self, length):
+        segment_ids = []
+        while True:
+            stats = self.get_stats()
+            segments = stats["partitions"][0]["segments"]
+            # random choice one segment
+            segment = random.choice(segments)
+            try:
+                segment_ids = self._milvus.list_id_in_segment(self._collection_name, segment["id"])
+            except Exception as e:
+                logger.error(str(e))
+            if not len(segment_ids):
+                continue
+            elif len(segment_ids) > length:
+                return random.sample(segment_ids, length)
+            else:
+                logger.debug("Reset length: %d" % len(segment_ids))
+                return segment_ids
+
+    # def get_rand_ids_each_segment(self, length):
+    #     res = []
+    #     status, stats = self._milvus.get_collection_stats(self._collection_name)
+    #     self.check_status(status)
+    #     segments = stats["partitions"][0]["segments"]
+    #     segments_num = len(segments)
+    #     # random choice from each segment
+    #     for segment in segments:
+    #         status, segment_ids = self._milvus.list_id_in_segment(self._collection_name, segment["name"])
+    #         self.check_status(status)
+    #         res.extend(segment_ids[:length])
+    #     return segments_num, res
+
+    # def get_rand_entities(self, length):
+    #     ids = self.get_rand_ids(length)
+    #     status, get_res = self._milvus.get_entity_by_id(self._collection_name, ids)
+    #     self.check_status(status)
+    #     return ids, get_res
+
+    @time_wrapper
+    def get_entities(self, get_ids):
+        get_res = self._milvus.get_entity_by_id(self._collection_name, get_ids)
+        return get_res
+
+    @time_wrapper
+    def delete(self, ids, collection_name=None):
+        tmp_collection_name = self._collection_name if collection_name is None else collection_name
+        self._milvus.delete_entity_by_id(tmp_collection_name, ids)
+
+    def delete_rand(self):
+        delete_id_length = random.randint(1, 100)
+        count_before = self.count()
+        logger.info("%s: length to delete: %d" % (self._collection_name, delete_id_length))
+        delete_ids = self.get_rand_ids(delete_id_length)
+        self.delete(delete_ids)
+        self.flush()
+        logger.info("%s: count after delete: %d" % (self._collection_name, self.count()))
+        get_res = self._milvus.get_entity_by_id(self._collection_name, delete_ids)
+        for item in get_res:
+            assert not item
+        # if count_before - len(delete_ids) < self.count():
+        #     logger.error(delete_ids)
+        #     raise Exception("Error occured")
+
+    @time_wrapper
+    def flush(self, collection_name=None):
+        tmp_collection_name = self._collection_name if collection_name is None else collection_name
+        self._milvus.flush([tmp_collection_name])
+
+    @time_wrapper
+    def compact(self, collection_name=None):
+        tmp_collection_name = self._collection_name if collection_name is None else collection_name
+        status = self._milvus.compact(tmp_collection_name)
         self.check_status(status)
 
     @time_wrapper
-    def insert(self, X, ids=None):
-        status, result = self._milvus.add_vectors(self._collection_name, X, ids)
-        self.check_status(status)
-        return status, result
-
-    @time_wrapper
-    def delete_vectors(self, ids):
-        status = self._milvus.delete_by_id(self._collection_name, ids)
-        self.check_status(status)
-
-    @time_wrapper
-    def flush(self):
-        status = self._milvus.flush([self._collection_name])
-        self.check_status(status)
-
-    @time_wrapper
-    def compact(self):
-        status = self._milvus.compact(self._collection_name)
-        self.check_status(status)
-
-    @time_wrapper
-    def create_index(self, index_type, index_param=None):
+    def create_index(self, field_name, index_type, metric_type, index_param=None):
         index_type = INDEX_MAP[index_type]
-        logger.info("Building index start, collection_name: %s, index_type: %s" % (self._collection_name, index_type))
+        metric_type = utils.metric_type_trans(metric_type)
+        logger.info("Building index start, collection_name: %s, index_type: %s, metric_type: %s" % (
+            self._collection_name, index_type, metric_type))
         if index_param:
             logger.info(index_param)
-        status = self._milvus.create_index(self._collection_name, index_type, index_param)
-        self.check_status(status)
+        index_params = {
+            "index_type": index_type,
+            "metric_type": metric_type,
+            "params": index_param
+        }
+        self._milvus.create_index(self._collection_name, field_name, index_params)
 
     def describe_index(self):
-        status, result = self._milvus.describe_index(self._collection_name)
-        self.check_status(status)
-        index_type = None
-        for k, v in INDEX_MAP.items():
-            if result._index_type == v:
-                index_type = k
-                break
-        return {"index_type": index_type, "index_param": result._params}
+        # stats = self.get_stats()
+        info = self.get_info()
+        index_info = {"index_type": "flat", "index_param": None}
+        for field in info["fields"]:
+            for index in field['indexes']:
+                if not index or "index_type" not in index:
+                    continue
+                else:
+                    for k, v in INDEX_MAP.items():
+                        if index['index_type'] == v:
+                            index_info['index_type'] = k
+                            index_info['index_param'] = index['params']
+                            return index_info
+        return index_info
 
-    def drop_index(self):
+    def drop_index(self, field_name):
         logger.info("Drop index: %s" % self._collection_name)
-        return self._milvus.drop_index(self._collection_name)
+        return self._milvus.drop_index(self._collection_name, field_name)
 
     @time_wrapper
-    def query(self, X, top_k, search_param=None):
-        status, result = self._milvus.search_vectors(self._collection_name, top_k, query_records=X, params=search_param)
-        self.check_status(status)
+    def query(self, vector_query, filter_query=None, collection_name=None):
+        tmp_collection_name = self._collection_name if collection_name is None else collection_name
+        must_params = [vector_query]
+        if filter_query:
+            must_params.extend(filter_query)
+        query = {
+            "bool": {"must": must_params}
+        }
+        result = self._milvus.search(tmp_collection_name, query)
+        # logging.info(len(result))
+        # logging.info(len(result[0]))
         return result
 
-    @time_wrapper
-    def query_ids(self, top_k, ids, search_param=None):
-        status, result = self._milvus.search_by_ids(self._collection_name, ids, top_k, params=search_param)
-        self.check_result_ids(result)
-        return result
+    def get_ids(self, result):
+        idss = result._entities.ids
+        ids = []
+        len_idss = len(idss)
+        len_r = len(result)
+        top_k = len_idss // len_r
+        for offset in range(0, len_idss, top_k):
+            ids.append(idss[offset: min(offset + top_k, len_idss)])
+        return ids
 
-    def count(self):
-        return self._milvus.count_collection(self._collection_name)[1]
+    def query_rand(self, nq_max=100):
+        # for ivf search
+        dimension = 128
+        top_k = random.randint(1, 100)
+        nq = random.randint(1, nq_max)
+        nprobe = random.randint(1, 100)
+        search_param = {"nprobe": nprobe}
+        query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)]
+        metric_type = random.choice(["l2", "ip"])
+        logger.info("%s, Search nq: %d, top_k: %d, nprobe: %d" % (self._collection_name, nq, top_k, nprobe))
+        vec_field_name = utils.get_default_field_name()
+        vector_query = {"vector": {vec_field_name: {
+            "topk": top_k,
+            "query": query_vectors,
+            "metric_type": utils.metric_type_trans(metric_type),
+            "params": search_param}
+        }}
+        self.query(vector_query)
 
-    def delete(self, timeout=120):
+    def count(self, collection_name=None):
+        if collection_name is None:
+            collection_name = self._collection_name
+        row_count = self._milvus.count_entities(collection_name)
+        logger.debug("Row count: %d in collection: <%s>" % (row_count, collection_name))
+        return row_count
+
+    def drop(self, timeout=120, collection_name=None):
         timeout = int(timeout)
-        logger.info("Start delete collection: %s" % self._collection_name)
-        self._milvus.drop_collection(self._collection_name)
+        if collection_name is None:
+            collection_name = self._collection_name
+        logger.info("Start delete collection: %s" % collection_name)
+        self._milvus.drop_collection(collection_name)
         i = 0
         while i < timeout:
-            if self.count():
-                time.sleep(1)
-                i = i + 1
-                continue
-            else:
+            try:
+                row_count = self.count(collection_name=collection_name)
+                if row_count:
+                    time.sleep(1)
+                    i = i + 1
+                    continue
+                else:
+                    break
+            except Exception as e:
+                logger.debug(str(e))
                 break
         if i >= timeout:
             logger.error("Delete collection timeout")
 
-    def describe(self):
-        return self._milvus.describe_collection(self._collection_name)
+    def get_stats(self):
+        return self._milvus.get_collection_stats(self._collection_name)
+
+    def get_info(self, collection_name=None):
+        # pdb.set_trace()
+        if collection_name is None:
+            collection_name = self._collection_name
+        return self._milvus.get_collection_info(collection_name)
 
     def show_collections(self):
-        return self._milvus.show_collections()
+        return self._milvus.list_collections()
 
     def exists_collection(self, collection_name=None):
         if collection_name is None:
             collection_name = self._collection_name
-        status, res = self._milvus.has_collection(collection_name)
-        # self.check_status(status)
+        res = self._milvus.has_collection(collection_name)
         return res
 
+    def clean_db(self):
+        collection_names = self.show_collections()
+        for name in collection_names:
+            self.drop(collection_name=name)
+
     @time_wrapper
     def preload_collection(self):
-        status = self._milvus.preload_collection(self._collection_name, timeout=3000)
-        self.check_status(status)
-        return status
+        return self._milvus.load_collection(self._collection_name, timeout=3000)
 
     def get_server_version(self):
-        status, res = self._milvus.server_version()
-        return res
+        return self._milvus.server_version()
 
     def get_server_mode(self):
         return self.cmd("mode")
@@ -207,166 +376,26 @@ class MilvusClient(object):
         return self.cmd("build_commit_id")
 
     def get_server_config(self):
-        return json.loads(self.cmd("get_config *"))
+        return json.loads(self.cmd("get_milvus_config"))
 
     def get_mem_info(self):
         result = json.loads(self.cmd("get_system_info"))
         result_human = {
             # unit: Gb
-            "memory_used": round(int(result["memory_used"]) / (1024*1024*1024), 2)
+            "memory_used": round(int(result["memory_used"]) / (1024 * 1024 * 1024), 2)
         }
         return result_human
 
     def cmd(self, command):
-        status, res = self._milvus._cmd(command)
+        res = self._milvus._cmd(command)
         logger.info("Server command: %s, result: %s" % (command, res))
-        self.check_status(status)
         return res
 
+    @time_wrapper
+    def set_config(self, parent_key, child_key, value):
+        self._milvus.set_config(parent_key, child_key, value)
 
-def fit(collection_name, X):
-    milvus = Milvus()
-    milvus.connect(host = SERVER_HOST_DEFAULT, port = SERVER_PORT_DEFAULT) 
-    start = time.time()
-    status, ids = milvus.add_vectors(collection_name, X)
-    end = time.time()
-    logger(status, round(end - start, 2))
-
-
-def fit_concurrent(collection_name, process_num, vectors):
-    processes = []
-
-    for i in range(process_num):
-        p = Process(target=fit, args=(collection_name, vectors, ))
-        processes.append(p)
-        p.start()
-    for p in processes:             
-        p.join()
-    
-
-if __name__ == "__main__":
-    import numpy
-    import sklearn.preprocessing
-
-    # collection_name = "tset_test"
-    # # collection_name = "test_tset1"
-    # m = MilvusClient(collection_name)
-    # m.delete()
-    # time.sleep(2)
-    # m.create_collection(collection_name, 128, 20, "ip")
-
-    # print(m.describe())
-    # print(m.count())
-    # print(m.describe_index())
-    # # sys.exit()
-    # tmp = [[random.random() for _ in range(128)] for _ in range(20000)]
-    # tmp1 = sklearn.preprocessing.normalize(tmp, axis=1, norm='l2')
-    # print(tmp1[0][0])
-    # tmp = [[random.random() for _ in range(128)] for _ in range(20000)]
-    # tmp /= numpy.linalg.norm(tmp)
-    # print(tmp[0][0])
-
-    # sum_1 = 0
-    # sum_2 = 0
-    # for item in tmp:
-    #     for i in item:
-    #         sum_2 = sum_2 + i * i
-    #     break
-    # for item in tmp1:
-    #     for i in item:
-    #         sum_1 = sum_1 + i * i
-    #     break
-    # print(sum_1, sum_2)
-    # insert_vectors = tmp.tolist()
-    # # print(insert_vectors)
-    # for i in range(2):
-    #     m.insert(insert_vectors)
-
-    # time.sleep(5)
-    # print(m.create_index("ivf_flat", 16384))
-    # X = [insert_vectors[0], insert_vectors[1], insert_vectors[2]]
-    # top_k = 5
-    # nprobe = 1
-    # print(m.query(X, top_k, nprobe))
-
-    # # print(m.drop_index())
-    # print(m.describe_index())
-    # sys.exit()
-    # # insert_vectors = [[random.random() for _ in range(128)] for _ in range(100000)]
-    # # for i in range(100):
-    # #     m.insert(insert_vectors)
-    # # time.sleep(5)
-    # # print(m.describe_index())
-    # # print(m.drop_index())
-    # m.create_index("ivf_sq8h", 16384)
-    # print(m.count())
-    # print(m.describe_index())
-
-
-
-    # sys.exit()
-    # print(m.create_index("ivf_sq8h", 16384))
-    # print(m.count())
-    # print(m.describe_index())
-    import numpy as np
-
-    # def mmap_fvecs(fname):
-    #     x = np.memmap(fname, dtype='int32', mode='r')
-    #     d = x[0]
-    #     return x.view('float32').reshape(-1, d + 1)[:, 1:]
-
-    # print(mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs"))
-    # SIFT_SRC_QUERY_DATA_DIR = '/poc/yuncong/ann_1000m'
-    # file_name = SIFT_SRC_QUERY_DATA_DIR+'/'+'query.npy'
-    # data = numpy.load(file_name)
-    # query_vectors = data[0:2].tolist()
-    # print(len(query_vectors))
-    # results = m.query(query_vectors, 10, 10)
-    # result_ids = []
-    # for result in results[1]:
-    #     tmp = []
-    #     for item in result:
-    #         tmp.append(item.id)
-    #     result_ids.append(tmp)
-    # print(result_ids[0][:10])
-    # # gt
-    # file_name = SIFT_SRC_QUERY_DATA_DIR+"/gnd/"+"idx_1M.ivecs"
-    # a = numpy.fromfile(file_name, dtype='int32')
-    # d = a[0]
-    # true_ids = a.reshape(-1, d + 1)[:, 1:].copy()
-    # print(true_ids[:3, :2])
-
-    # print(len(true_ids[0]))
-    # import numpy as np
-    # import sklearn.preprocessing
-
-    # def mmap_fvecs(fname):
-    #     x = np.memmap(fname, dtype='int32', mode='r')
-    #     d = x[0]
-    #     return x.view('float32').reshape(-1, d + 1)[:, 1:]
-
-    # data = mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs")
-    # data = sklearn.preprocessing.normalize(data, axis=1, norm='l2')
-    # np.save("/test/milvus/deep1b/query.npy", data)
-    dimension = 4096
-    insert_xb = 10000
-    insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)]
-    data = sklearn.preprocessing.normalize(insert_vectors, axis=1, norm='l2')
-    np.save("/test/milvus/raw_data/random/query_%d.npy" % dimension, data)
-    sys.exit()
-
-    total_size = 100000000
-    # total_size = 1000000000
-    file_size = 100000
-    # file_size = 100000
-    dimension = 4096
-    file_num = total_size // file_size
-    for i in range(file_num):
-        print(i)
-        # fname = "/test/milvus/raw_data/deep1b/binary_96_%05d" % i
-        fname = "/test/milvus/raw_data/random/binary_%dd_%05d" % (dimension, i)
-        # print(fname, i*file_size, (i+1)*file_size)
-        # single_data = data[i*file_size : (i+1)*file_size]
-        single_data = [[random.random() for _ in range(dimension)] for _ in range(file_size)]
-        single_data = sklearn.preprocessing.normalize(single_data, axis=1, norm='l2')
-        np.save(fname, single_data)
+    def create_partition(self, tag, collection_name=None):
+        if not collection_name:
+            collection_name = self._collection_name
+        self._milvus.create_partition(collection_name, tag)

tests/milvus_benchmark/conf/log_config.conf

@@ -1,28 +0,0 @@
-* GLOBAL:
-    FORMAT                  =   "%datetime | %level | %logger | %msg"
-    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-global.log"
-    ENABLED                 =   true
-    TO_FILE                 =   true
-    TO_STANDARD_OUTPUT      =   false
-    SUBSECOND_PRECISION     =   3
-    PERFORMANCE_TRACKING    =   false
-    MAX_LOG_FILE_SIZE       =   2097152 ## Throw log files away after 2MB
-* DEBUG:
-    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-debug.log"
-    ENABLED                 =   true
-* WARNING:
-    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-warning.log"
-* TRACE:
-    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-trace.log"
-* VERBOSE:
-    FORMAT                  =   "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
-    TO_FILE                 =   false
-    TO_STANDARD_OUTPUT      =   false
-## Error logs
-* ERROR:
-    ENABLED                 =   true
-    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-error.log"
-* FATAL:
-    ENABLED                 =   true
-    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-fatal.log"
-

tests/milvus_benchmark/conf/server_config.yaml

@@ -1,34 +0,0 @@
-server_config:
-  address: 0.0.0.0                  
-  port: 19530                       
-  deploy_mode: single               
-  time_zone: UTC+8              
-
-db_config:
-  primary_path: /var/lib/milvus        
-  secondary_path:                   
-  backend_url: sqlite://:@:/      
-  insert_buffer_size: 4             
-  preload_collection:
-
-metric_config:
-  enable_monitor: true            
-  collector: prometheus      
-  prometheus_config:
-    port: 8080                   
-
-cache_config:
-  cpu_cache_capacity: 16          
-  cache_insert_data: false   
-
-engine_config:
-  use_blas_threshold: 1100          
-  gpu_search_threshold: 1000        
-gpu_resource_config:
-  enable: true                 
-  cache_capacity: 4                 
-  search_resources:              
-    - gpu0
-  build_index_resources:          
-    - gpu0
-

tests/milvus_benchmark/conf/server_config.yaml.cpu

@@ -1,31 +0,0 @@
-server_config:
-  address: 0.0.0.0            
-  port: 19530                 
-  deploy_mode: single                
-  time_zone: UTC+8  
-            
-db_config:
-  primary_path: /opt/milvus             
-  secondary_path:                        
-  backend_url: sqlite://:@:/    
-  insert_buffer_size: 4               
-  build_index_gpu: 0  
-  preload_collection:
-
-metric_config:
-  enable_monitor: false             
-  collector: prometheus             
-  prometheus_config:
-    port: 8080        
-
-cache_config:
-  cpu_cache_capacity: 16           
-  cpu_cache_threshold: 0.85     
-  cache_insert_data: false 
-
-engine_config:
-  use_blas_threshold: 20            
-
-resource_config:
-  resource_pool:
-    - cpu

tests/milvus_benchmark/conf/server_config.yaml.multi

@@ -1,33 +0,0 @@
-server_config:
-  address: 0.0.0.0            
-  port: 19530                 
-  deploy_mode: single                
-  time_zone: UTC+8  
-            
-db_config:
-  primary_path: /opt/milvus             
-  secondary_path:                        
-  backend_url: sqlite://:@:/    
-  insert_buffer_size: 4               
-  build_index_gpu: 0  
-  preload_collection:
-
-metric_config:
-  enable_monitor: false             
-  collector: prometheus             
-  prometheus_config:
-    port: 8080        
-
-cache_config:
-  cpu_cache_capacity: 16           
-  cpu_cache_threshold: 0.85     
-  cache_insert_data: false 
-
-engine_config:
-  use_blas_threshold: 20            
-
-resource_config:
-  resource_pool:
-    - cpu
-    - gpu0
-    - gpu1

tests/milvus_benchmark/conf/server_config.yaml.single

@@ -1,32 +0,0 @@
-server_config:
-  address: 0.0.0.0            
-  port: 19530                 
-  deploy_mode: single                
-  time_zone: UTC+8  
-            
-db_config:
-  primary_path: /opt/milvus             
-  secondary_path:                        
-  backend_url: sqlite://:@:/    
-  insert_buffer_size: 4               
-  build_index_gpu: 0  
-  preload_collection:
-
-metric_config:
-  enable_monitor: false             
-  collector: prometheus             
-  prometheus_config:
-    port: 8080        
-
-cache_config:
-  cpu_cache_capacity: 16           
-  cpu_cache_threshold: 0.85     
-  cache_insert_data: false 
-
-engine_config:
-  use_blas_threshold: 20            
-
-resource_config:
-  resource_pool:
-    - cpu
-    - gpu0

tests/milvus_benchmark/docker_utils.py

@@ -0,0 +1,125 @@
+# def pull_image(image):
+#     registry = image.split(":")[0]
+#     image_tag = image.split(":")[1]
+#     client = docker.APIClient(base_url='unix://var/run/docker.sock')
+#     logger.info("Start pulling image: %s" % image)
+#     return client.pull(registry, image_tag)
+
+
+# def run_server(image, mem_limit=None, timeout=30, test_type="local", volume_name=None, db_slave=None):
+#     import colors
+
+#     client = docker.from_env()
+#     # if mem_limit is None:
+#     #     mem_limit = psutil.virtual_memory().available
+#     # logger.info('Memory limit:', mem_limit)
+#     # cpu_limit = "0-%d" % (multiprocessing.cpu_count() - 1)
+#     # logger.info('Running on CPUs:', cpu_limit)
+#     for dir_item in ['logs', 'db']:
+#         try:
+#             os.mkdir(os.path.abspath(dir_item))
+#         except Exception as e:
+#             pass
+
+#     if test_type == "local":
+#         volumes = {
+#             os.path.abspath('conf'):
+#                 {'bind': '/opt/milvus/conf', 'mode': 'ro'},
+#             os.path.abspath('logs'):
+#                 {'bind': '/opt/milvus/logs', 'mode': 'rw'},
+#             os.path.abspath('db'):
+#                 {'bind': '/opt/milvus/db', 'mode': 'rw'},
+#         }
+#     elif test_type == "remote":
+#         if volume_name is None:
+#             raise Exception("No volume name")
+#         remote_log_dir = volume_name+'/logs'
+#         remote_db_dir = volume_name+'/db'
+
+#         for dir_item in [remote_log_dir, remote_db_dir]:
+#             if not os.path.isdir(dir_item):
+#                 os.makedirs(dir_item, exist_ok=True)
+#         volumes = {
+#             os.path.abspath('conf'):
+#                 {'bind': '/opt/milvus/conf', 'mode': 'ro'},
+#             remote_log_dir:
+#                 {'bind': '/opt/milvus/logs', 'mode': 'rw'},
+#             remote_db_dir:
+#                 {'bind': '/opt/milvus/db', 'mode': 'rw'}
+#         }
+#         # add volumes
+#         if db_slave and isinstance(db_slave, int):
+#             for i in range(2, db_slave+1):
+#                 remote_db_dir = volume_name+'/data'+str(i)
+#                 if not os.path.isdir(remote_db_dir):
+#                     os.makedirs(remote_db_dir, exist_ok=True)
+#                 volumes[remote_db_dir] = {'bind': '/opt/milvus/data'+str(i), 'mode': 'rw'}
+
+#     container = client.containers.run(
+#         image,
+#         volumes=volumes,
+#         runtime="nvidia",
+#         ports={'19530/tcp': 19530, '8080/tcp': 8080},
+#         # environment=["OMP_NUM_THREADS=48"],
+#         # cpuset_cpus=cpu_limit,
+#         # mem_limit=mem_limit,
+#         # environment=[""],
+#         detach=True)
+
+#     def stream_logs():
+#         for line in container.logs(stream=True):
+#             logger.info(colors.color(line.decode().rstrip(), fg='blue'))
+
+#     if sys.version_info >= (3, 0):
+#         t = threading.Thread(target=stream_logs, daemon=True)
+#     else:
+#         t = threading.Thread(target=stream_logs)
+#         t.daemon = True
+#     t.start()
+
+#     logger.info('Container: %s started' % container)
+#     return container
+#     # exit_code = container.wait(timeout=timeout)
+#     # # Exit if exit code
+#     # if exit_code == 0:
+#     #     return container
+#     # elif exit_code is not None:
+#     #     print(colors.color(container.logs().decode(), fg='red'))
+
+# def restart_server(container):
+#     client = docker.APIClient(base_url='unix://var/run/docker.sock')
+
+#     client.restart(container.name)
+#     logger.info('Container: %s restarted' % container.name)
+#     return container
+
+
+# def remove_container(container):
+#     container.remove(force=True)
+#     logger.info('Container: %s removed' % container)
+
+
+# def remove_all_containers(image):
+#     client = docker.from_env()
+#     try:
+#         for container in client.containers.list():
+#             if image in container.image.tags:
+#                 container.stop(timeout=30)
+#                 container.remove(force=True)
+#     except Exception as e:
+#         logger.error("Containers removed failed")
+
+
+# def container_exists(image):
+#     '''
+#     Check if container existed with the given image name
+#     @params: image name
+#     @return: container if exists
+#     '''
+#     res = False
+#     client = docker.from_env()
+#     for container in client.containers.list():
+#         if image in container.image.tags:
+#             # True
+#             res = container
+#     return res

tests/milvus_benchmark/executors/__init__.py

@@ -0,0 +1,3 @@
+
+class BaseExecutor(object):
+    pass

tests/milvus_benchmark/executors/shell.py

@@ -0,0 +1,4 @@
+from . import BaseExecutor
+
+class ShellExecutor(BaseExecutor):
+    pass

tests/milvus_benchmark/helm_utils.py

@@ -0,0 +1,326 @@
+import os
+import pdb
+import time
+import logging
+from yaml import full_load, dump
+import utils
+
+logger = logging.getLogger("milvus_benchmark.utils")
+REGISTRY_URL = "registry.zilliz.com/milvus/engine"
+
+
+def retry(times):
+    """
+    This decorator prints the execution time for the decorated function.
+    """
+    def wrapper(func):
+        def newfn(*args, **kwargs):
+            attempt = 0
+            while attempt < times:
+                try:
+                    logger.info(attempt)
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    logger.info(str(e))
+                    time.sleep(3)
+                    attempt += 1
+            return func(*args, **kwargs)
+        return newfn
+    return wrapper
+
+
+@utils.retry(3)
+def get_host_cpus(hostname):
+    from kubernetes import client, config
+    config.load_kube_config()
+    client.rest.logger.setLevel(logging.WARNING)
+    v1 = client.CoreV1Api()
+    cpus = v1.read_node(hostname).status.allocatable.get("cpu")
+    return cpus
+
+
+# update values.yaml
+def update_values(file_path, deploy_mode, hostname, milvus_config, server_config=None):
+    if not os.path.isfile(file_path):
+        raise Exception('File: %s not found' % file_path)
+    # 　bak values.yaml
+    file_name = os.path.basename(file_path)
+    bak_file_name = file_name + ".bak"
+    file_parent_path = os.path.dirname(file_path)
+    bak_file_path = file_parent_path + '/' + bak_file_name
+    if os.path.exists(bak_file_path):
+        os.system("cp %s %s" % (bak_file_path, file_path))
+    else:
+        os.system("cp %s %s" % (file_path, bak_file_path))
+    with open(file_path) as f:
+        values_dict = full_load(f)
+        f.close()
+
+    for k, v in milvus_config.items():
+        if k.find("primary_path") != -1:
+            suffix_path = milvus_config["suffix_path"] if "suffix_path" in milvus_config else None
+            path_value = v
+            if suffix_path:
+                path_value = v + "_" + str(int(time.time()))
+            values_dict["primaryPath"] = path_value
+            values_dict['wal']['path'] = path_value + "/wal"
+            values_dict['logs']['path'] = path_value + "/logs"
+        # elif k.find("use_blas_threshold") != -1:
+        #     values_dict['useBLASThreshold'] = int(v)
+        elif k.find("gpu_search_threshold") != -1:
+            values_dict['gpu']['gpuSearchThreshold'] = int(v)
+        elif k.find("cpu_cache_capacity") != -1:
+            values_dict['cache']['cacheSize'] = v
+        # elif k.find("cache_insert_data") != -1:
+        #     values_dict['cache']['cacheInsertData'] = v
+        elif k.find("insert_buffer_size") != -1:
+            values_dict['cache']['insertBufferSize'] = v
+        elif k.find("gpu_resource_config.enable") != -1:
+            values_dict['gpu']['enabled'] = v
+        elif k.find("gpu_resource_config.cache_capacity") != -1:
+            values_dict['gpu']['cacheSize'] = v
+        elif k.find("build_index_resources") != -1:
+            values_dict['gpu']['buildIndexDevices'] = v
+        elif k.find("search_resources") != -1:
+            values_dict['gpu']['searchDevices'] = v
+        # wal
+        elif k.find("auto_flush_interval") != -1:
+            values_dict['storage']['autoFlushInterval'] = v
+        elif k.find("wal_enable") != -1:
+            values_dict['wal']['enabled'] = v
+
+    # if values_dict['nodeSelector']:
+    #     logger.warning("nodeSelector has been set: %s" % str(values_dict['engine']['nodeSelector']))
+    #     return
+    values_dict["wal"]["recoveryErrorIgnore"] = True
+    # enable monitor
+    values_dict["metrics"]["enabled"] = True
+    values_dict["metrics"]["address"] = "192.168.1.237"
+    values_dict["metrics"]["port"] = 9091
+    # Using sqlite for single mode
+    if deploy_mode == "single":
+        values_dict["mysql"]["enabled"] = False
+
+    # update values.yaml with the given host
+    if hostname:
+        values_dict['nodeSelector'] = {'kubernetes.io/hostname': hostname}
+        cpus = server_config["cpus"]
+
+        # set limit/request cpus in resources
+        values_dict["image"]['resources'] = {
+            "limits": {
+                # "cpu": str(int(cpus)) + ".0"
+                "cpu": str(int(cpus)) + ".0"
+            },
+            "requests": {
+                "cpu": str(int(cpus) - 1) + ".0"
+            }
+        }
+        values_dict['tolerations'] = [{
+            "key": "worker",
+            "operator": "Equal",
+            "value": "performance",
+            "effect": "NoSchedule"
+        }]
+    # add extra volumes
+    values_dict['extraVolumes'] = [{
+        'name': 'test',
+        'flexVolume': {
+            'driver': "fstab/cifs",
+            'fsType': "cifs",
+            'secretRef': {
+                'name': "cifs-test-secret"
+            },
+            'options': {
+                'networkPath': "//192.168.1.126/test",
+                'mountOptions': "vers=1.0"
+            }
+        }
+    }]
+    values_dict['extraVolumeMounts'] = [{
+        'name': 'test',
+        'mountPath': '/test'
+    }]
+
+    # add extra volumes for mysql
+    # values_dict['mysql']['persistence']['enabled'] = True
+    # values_dict['mysql']['configurationFilesPath'] = "/etc/mysql/mysql.conf.d/"
+    # values_dict['mysql']['imageTag'] = '5.6'
+    # values_dict['mysql']['securityContext'] = {
+    #         'enabled': True}
+    # mysql_db_path = "/test"
+    if deploy_mode == "shards":
+        # mount_path = values_dict["primaryPath"]+'/data'
+        # long_str = '- name: test-mysql\n  flexVolume:\n    driver: fstab/cifs\n    fsType: cifs\n    secretRef:\n      name: cifs-test-secret\n    options:\n      networkPath: //192.168.1.126/test\n      mountOptions: vers=1.0'
+        # values_dict['mysql']['extraVolumes'] = literal_str(long_str)
+        # long_str_2 = "- name: test-mysql\n  mountPath: %s" % mysql_db_path
+        # values_dict['mysql']['extraVolumeMounts'] = literal_str(long_str_2)
+        # mysql_cnf_str = '[mysqld]\npid-file=%s/mysql.pid\ndatadir=%s' % (mount_path, mount_path)
+        # values_dict['mysql']['configurationFiles'] = {}
+        # values_dict['mysql']['configurationFiles']['mysqld.cnf'] = literal_str(mysql_cnf_str)
+
+        values_dict['mysql']['enabled'] = False
+        values_dict['externalMysql']['enabled'] = True
+        values_dict['externalMysql']["ip"] = "192.168.1.197"
+        values_dict['externalMysql']["port"] = 3306
+        values_dict['externalMysql']["user"] = "root"
+        values_dict['externalMysql']["password"] = "Fantast1c"
+        values_dict['externalMysql']["database"] = "db"
+
+    # logger.debug(values_dict)
+    #  print(dump(values_dict))
+    with open(file_path, 'w') as f:
+        dump(values_dict, f, default_flow_style=False)
+    f.close()
+    # DEBUG
+    with open(file_path) as f:
+        for line in f.readlines():
+            line = line.strip("\n")
+
+
+# deploy server
+def helm_install_server(helm_path, deploy_mode, image_tag, image_type, name, namespace):
+    timeout = 300
+    logger.debug("Server deploy mode: %s" % deploy_mode)
+    host = "%s.%s.svc.cluster.local" % (name, namespace)
+    if deploy_mode == "single":
+        install_cmd = "helm install --wait --timeout %ds \
+                --set image.repository=%s \
+                --set image.tag=%s \
+                --set image.pullPolicy=Always \
+                --set service.type=ClusterIP \
+                -f ci/filebeat/values.yaml \
+                --namespace %s \
+                %s ." % (timeout, REGISTRY_URL, image_tag, namespace, name)
+    elif deploy_mode == "shards":
+        install_cmd = "helm install --wait --timeout %ds \
+                --set cluster.enabled=true \
+                --set persistence.enabled=true \
+                --set mishards.image.tag=test \
+                --set mishards.image.pullPolicy=Always \
+                --set image.repository=%s \
+                --set image.tag=%s \
+                --set image.pullPolicy=Always \
+                --set service.type=ClusterIP \
+                -f ci/filebeat/values.yaml \
+                --namespace %s \
+                %s ." % (timeout, REGISTRY_URL, image_tag, namespace, name)
+    logger.debug(install_cmd)
+    logger.debug(host)
+    if os.system("cd %s && %s" % (helm_path, install_cmd)):
+        logger.error("Helm install failed: %s" % name)
+        return None
+    time.sleep(5)
+    # config.load_kube_config()
+    # v1 = client.CoreV1Api()
+    # pod_name = None
+    # pod_id = None
+    # pods = v1.list_namespaced_pod(namespace)
+    # for i in pods.items:
+    #     if i.metadata.name.find(name) != -1:
+    #         pod_name = i.metadata.name
+    #         pod_ip = i.status.pod_ip
+    # logger.debug(pod_name)
+    # logger.debug(pod_ip)
+    # return pod_name, pod_ip
+    return host
+
+
+# delete server
+def helm_del_server(name, namespace):
+    # logger.debug("Sleep 600s before uninstall server")
+    # time.sleep(600)
+    del_cmd = "helm uninstall -n milvus %s" % name
+    logger.debug(del_cmd)
+    if os.system(del_cmd):
+        logger.error("Helm delete name:%s failed" % name)
+        return False
+    return True
+
+
+def restart_server(helm_release_name, namespace):
+    res = True
+    timeout = 120000
+    # service_name = "%s.%s.svc.cluster.local" % (helm_release_name, namespace)
+    config.load_kube_config()
+    v1 = client.CoreV1Api()
+    pod_name = None
+    # config_map_names = v1.list_namespaced_config_map(namespace, pretty='true')
+    # body = {"replicas": 0}
+    pods = v1.list_namespaced_pod(namespace)
+    for i in pods.items:
+        if i.metadata.name.find(helm_release_name) != -1 and i.metadata.name.find("mysql") == -1:
+            pod_name = i.metadata.name
+            break
+            # v1.patch_namespaced_config_map(config_map_name, namespace, body, pretty='true')
+    # status_res = v1.read_namespaced_service_status(helm_release_name, namespace, pretty='true')
+    logger.debug("Pod name: %s" % pod_name)
+    if pod_name is not None:
+        try:
+            v1.delete_namespaced_pod(pod_name, namespace)
+        except Exception as e:
+            logging.error(str(e))
+            logging.error("Exception when calling CoreV1Api->delete_namespaced_pod")
+            res = False
+            return res
+        logging.error("Sleep 10s after pod deleted")
+        time.sleep(10)
+        # check if restart successfully
+        pods = v1.list_namespaced_pod(namespace)
+        for i in pods.items:
+            pod_name_tmp = i.metadata.name
+            logging.error(pod_name_tmp)
+            if pod_name_tmp == pod_name:
+                continue
+            elif pod_name_tmp.find(helm_release_name) == -1 or pod_name_tmp.find("mysql") != -1:
+                continue
+            else:
+                status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true')
+                logging.error(status_res.status.phase)
+                start_time = time.time()
+                ready_break = False
+                while time.time() - start_time <= timeout:
+                    logging.error(time.time())
+                    status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true')
+                    if status_res.status.phase == "Running":
+                        logging.error("Already running")
+                        ready_break = True
+                        break
+                    else:
+                        time.sleep(5)
+                if time.time() - start_time > timeout:
+                    logging.error("Restart pod: %s timeout" % pod_name_tmp)
+                    res = False
+                    return res
+                if ready_break:
+                    break
+    else:
+        raise Exception("Pod: %s not found" % pod_name)
+    follow = True
+    pretty = True
+    previous = True  # bool | Return previous terminated container logs. Defaults to false. (optional)
+    since_seconds = 56  # int | A relative time in seconds before the current time from which to show logs. If this value precedes the time a pod was started, only logs since the pod start will be returned. If this value is in the future, no logs will be returned. Only one of sinceSeconds or sinceTime may be specified. (optional)
+    timestamps = True  # bool | If true, add an RFC3339 or RFC3339Nano timestamp at the beginning of every line of log output. Defaults to false. (optional)
+    container = "milvus"
+    # start_time = time.time()
+    # while time.time() - start_time <= timeout:
+    #     try:
+    #         api_response = v1.read_namespaced_pod_log(pod_name_tmp, namespace, container=container, follow=follow,
+    #                                                 pretty=pretty, previous=previous, since_seconds=since_seconds,
+    #                                                 timestamps=timestamps)
+    #         logging.error(api_response)
+    #         return res
+    #     except Exception as e:
+    #         logging.error("Exception when calling CoreV1Api->read_namespaced_pod_log: %s\n" % e)
+    #         # waiting for server start
+    #         time.sleep(2)
+    #         # res = False
+    #         # return res
+    # if time.time() - start_time > timeout:
+    #     logging.error("Restart pod: %s timeout" % pod_name_tmp)
+    #     res = False
+    return res
+
+
+if __name__ == '__main__':
+    update_values("", "shards", None, None)

tests/milvus_benchmark/local_runner.py

@@ -1,16 +1,23 @@
 import os
 import logging
 import pdb
+import string
 import time
 import random
 import json
+import csv
 from multiprocessing import Process
 import numpy as np
 import concurrent.futures
+from queue import Queue
+
+import locust_user
+from milvus import DataType
 from client import MilvusClient
+from runner import Runner
 import utils
 import parser
-from runner import Runner
+
 
 DELETE_INTERVAL_TIME = 5
 INSERT_INTERVAL = 50000
@@ -19,92 +26,135 @@ logger = logging.getLogger("milvus_benchmark.local_runner")
 
 class LocalRunner(Runner):
     """run local mode"""
-    def __init__(self, ip, port):
+    def __init__(self, host, port):
         super(LocalRunner, self).__init__()
-        self.ip = ip
+        self.host = host
         self.port = port
 
     def run(self, run_type, collection):
         logger.debug(run_type)
         logger.debug(collection)
-        collection_name = collection["collection_name"]
-        milvus_instance = MilvusClient(collection_name=collection_name, ip=self.ip, port=self.port)
+        collection_name = collection["collection_name"] if "collection_name" in collection else None
+        milvus_instance = MilvusClient(collection_name=collection_name, host=self.host, port=self.port)
         logger.info(milvus_instance.show_collections())
-        env_value = milvus_instance.get_server_config()
-        logger.debug(env_value)
+        # TODO:
+        self.env_value = milvus_instance.get_server_config()
+        # ugly implemention
+        self.env_value = utils.convert_nested(self.env_value)
+        self.env_value.pop("logs")
+        self.env_value.pop("network")
+        logger.info(self.env_value)
 
         if run_type in ["insert_performance", "insert_flush_performance"]:
-            (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
+            (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name)
             ni_per = collection["ni_per"]
             build_index = collection["build_index"]
             if milvus_instance.exists_collection():
-                milvus_instance.delete()
+                milvus_instance.drop()
                 time.sleep(10)
-            milvus_instance.create_collection(collection_name, dimension, index_file_size, metric_type)
+            vector_type = self.get_vector_type(data_type)
+            other_fields = collection["other_fields"] if "other_fields" in collection else None
+            milvus_instance.create_collection(dimension, segment_row_limit, data_type=vector_type, other_fields=other_fields)
             if build_index is True:
                 index_type = collection["index_type"]
                 index_param = collection["index_param"]
-                milvus_instance.create_index(index_type, index_param)
-                logger.debug(milvus_instance.describe_index())
+                index_field_name = utils.get_default_field_name(vector_type)
+                milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param)
             res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per)
             milvus_instance.flush()
             logger.debug("Table row counts: %d" % milvus_instance.count())
             if build_index is True:
                 logger.debug("Start build index for last file")
-                milvus_instance.create_index(index_type, index_param)
-                logger.debug(milvus_instance.describe_index())
+                milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param)
 
         elif run_type == "delete_performance":
-            (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
+            (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name)
             ni_per = collection["ni_per"]
+            auto_flush = collection["auto_flush"] if "auto_flush" in collection else True
             if not milvus_instance.exists_collection():
                 logger.error(milvus_instance.show_collections())
-                logger.warning("Table: %s not found" % collection_name)
+                logger.error("Table: %s not found" % collection_name)
                 return
             length = milvus_instance.count() 
             ids = [i for i in range(length)] 
             loops = int(length / ni_per)
+            if auto_flush is False:
+                milvus_instance.set_config("storage", "auto_flush_interval", BIG_FLUSH_INTERVAL)
             for i in range(loops):
-                delete_ids = ids[i*ni_per : i*ni_per+ni_per]
+                delete_ids = ids[i*ni_per: i*ni_per+ni_per]
                 logger.debug("Delete %d - %d" % (delete_ids[0], delete_ids[-1]))
-                milvus_instance.delete_vectors(delete_ids)
-                milvus_instance.flush() 
+                milvus_instance.delete(delete_ids)
                 logger.debug("Table row counts: %d" % milvus_instance.count())
             logger.debug("Table row counts: %d" % milvus_instance.count())
-            milvus_instance.flush() 
+            milvus_instance.flush()
+            logger.debug("Table row counts: %d" % milvus_instance.count())
+
+        elif run_type == "compact_performance":
+            (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name)
+            if not milvus_instance.exists_collection():
+                logger.error(milvus_instance.show_collections())
+                logger.error("Table: %s not found" % collection_name)
+                return
+            length = milvus_instance.count()
+            ids = [i for i in range(length)]
+            loops = int(length / segment_row_limit)
+            delete_ni_per = segment_row_limit // 2
+            ni_per = segment_row_limit
+            for i in range(loops):
+                delete_ids = ids[i*ni_per : i*ni_per+delete_ni_per]
+                logger.debug("Delete %d - %d" % (delete_ids[0], delete_ids[-1]))
+                milvus_instance.delete(delete_ids)
+                milvus_instance.flush()
+                logger.debug("Table row counts: %d" % milvus_instance.count())
+            logger.debug("Table row counts: %d" % milvus_instance.count())
+            milvus_instance.flush()
+            milvus_instance.compact()
             logger.debug("Table row counts: %d" % milvus_instance.count())
 
         elif run_type == "build_performance":
-            (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
+            (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name)
             index_type = collection["index_type"]
             index_param = collection["index_param"]
             if not milvus_instance.exists_collection():
                 logger.error("Table name: %s not existed" % collection_name)
                 return
-            search_params = {}
-            start_time = time.time()
+            vector_type = self.get_vector_type(data_type)
+            index_field_name = utils.get_default_field_name(vector_type)
             # drop index
             logger.debug("Drop index")
-            milvus_instance.drop_index()
+            milvus_instance.drop_index(index_field_name)
             start_mem_usage = milvus_instance.get_mem_info()["memory_used"]
-            milvus_instance.create_index(index_type, index_param)
-            logger.debug(milvus_instance.describe_index())
-            logger.debug("Table row counts: %d" % milvus_instance.count())
+            start_time = time.time()
+            milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param)
             end_time = time.time()
+            logger.debug("Table row counts: %d" % milvus_instance.count())
             end_mem_usage = milvus_instance.get_mem_info()["memory_used"]
             logger.debug("Diff memory: %s, current memory usage: %s, build time: %s" % ((end_mem_usage - start_mem_usage), end_mem_usage, round(end_time - start_time, 1)))
 
         elif run_type == "search_performance":
-            (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
+            (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name)
             run_count = collection["run_count"]
             top_ks = collection["top_ks"]
             nqs = collection["nqs"]
             search_params = collection["search_params"]
+            filter_query = []
+            filters = collection["filters"] if "filters" in collection else []
+            # pdb.set_trace()
+            # ranges = collection["range"] if "range" in collection else None
+            # terms = collection["term"] if "term" in collection else None
+            # if ranges:
+            #     filter_query.append(eval(ranges))
+            # if terms:
+            #     filter_query.append(eval(terms))
+            vector_type = self.get_vector_type(data_type)
+            vec_field_name = utils.get_default_field_name(vector_type)
             # for debugging
             # time.sleep(3600)
             if not milvus_instance.exists_collection():
                 logger.error("Table name: %s not existed" % collection_name)
                 return
+            vector_type = self.get_vector_type(data_type)
+            vec_field_name = utils.get_default_field_name(vector_type)
             logger.info(milvus_instance.count())
             result = milvus_instance.describe_index()
             logger.info(result)
@@ -113,16 +163,90 @@ class LocalRunner(Runner):
             logger.info(mem_usage)
             for search_param in search_params:
                 logger.info("Search param: %s" % json.dumps(search_param))
-                res = self.do_query(milvus_instance, collection_name, top_ks, nqs, run_count, search_param)
-                headers = ["Nq/Top-k"]
-                headers.extend([str(top_k) for top_k in top_ks])
-                logger.info("Search param: %s" % json.dumps(search_param))
-                utils.print_table(headers, nqs, res)
-                mem_usage = milvus_instance.get_mem_info()["memory_used"]
-                logger.info(mem_usage)
+                filter_param = []
+                if not filters:
+                    filters.append(None)
+                for filter in filters:
+                    if isinstance(filter, dict) and "range" in filter:
+                        filter_query.append(eval(filter["range"]))
+                        filter_param.append(filter["range"])
+                    if isinstance(filter, dict) and "term" in filter:
+                        filter_query.append(eval(filter["term"]))
+                        filter_param.append(filter["term"])
+                    logger.info("filter param: %s" % json.dumps(filter_param))
+                    res = self.do_query(milvus_instance, collection_name, vec_field_name, top_ks, nqs, run_count, search_param, filter_query)
+                    headers = ["Nq/Top-k"]
+                    headers.extend([str(top_k) for top_k in top_ks])
+                    logger.info("Search param: %s" % json.dumps(search_param))
+                    utils.print_table(headers, nqs, res)
+                    mem_usage = milvus_instance.get_mem_info()["memory_used"]
+                    logger.info(mem_usage)
+
+        elif run_type == "locust_search_performance":
+            (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name)
+            ni_per = collection["ni_per"]
+            build_index = collection["build_index"]
+            vector_type = self.get_vector_type(data_type)
+            index_field_name = utils.get_default_field_name(vector_type)
+            if build_index is True:
+                index_type = collection["index_type"]
+                index_param = collection["index_param"]
+            # TODO: debug
+            # if milvus_instance.exists_collection():
+            #     milvus_instance.drop()
+            #     time.sleep(10)
+            # other_fields = collection["other_fields"] if "other_fields" in collection else None
+            # milvus_instance.create_collection(dimension, segment_row_limit, data_type=vector_type, other_fields=other_fields)
+            # milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param)
+            # res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per)
+            # milvus_instance.flush()
+            # logger.debug("Table row counts: %d" % milvus_instance.count())
+            # if build_index is True:
+            #     logger.debug("Start build index for last file")
+            #     milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param)
+            real_metric_type = utils.metric_type_trans(metric_type)
+            ### spawn locust requests
+            task = collection["task"]
+            connection_type = "single"
+            connection_num = task["connection_num"]
+            if connection_num > 1:
+                connection_type = "multi"
+            clients_num = task["clients_num"]
+            hatch_rate = task["hatch_rate"]
+            during_time = utils.timestr_to_int(task["during_time"])
+            task_types = task["types"]
+            # """
+            # task: 
+            #     connection_num: 1
+            #     clients_num: 100
+            #     hatch_rate: 2
+            #     during_time: 5m
+            #     types:
+            #     -
+            #         type: query
+            #         weight: 1
+            #         params:
+            #         top_k: 10
+            #         nq: 1
+            #         # filters:
+            #         #   -
+            #         #     range:
+            #         #       int64:
+            #         #         LT: 0
+            #         #         GT: 1000000
+            #         search_param:
+            #             nprobe: 16
+            # """
+            run_params = {"tasks": {}, "clients_num": clients_num, "spawn_rate": hatch_rate, "during_time": during_time}
+            for task_type in task_types:
+                run_params["tasks"].update({task_type["type"]: task_type["weight"] if "weight" in task_type else 1})
+
+            #. collect stats
+            locust_stats = locust_user.locust_executor(self.host, self.port, collection_name, run_params=run_params)
+            logger.info(locust_stats)
 
         elif run_type == "search_ids_stability":
-            (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
+            (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name)
             search_params = collection["search_params"]
             during_time = collection["during_time"]
             ids_length = collection["ids_length"]
@@ -188,12 +312,12 @@ class LocalRunner(Runner):
                         logger.info("Search param: %s" % json.dumps(search_param))
                         total_time = 0.0
                         if use_single_connection is True:
-                            connections = [MilvusClient(collection_name=collection_name, ip=self.ip, port=self.port)]
+                            connections = [MilvusClient(collection_name=collection_name, host=self.host, port=self.port)]
                             with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor:
                                 future_results = {executor.submit(
                                     self.do_query_qps, connections[0], query_vectors, top_k, search_param=search_param) : index for index in range(concurrent_num)}
                         else:
-                            connections = [MilvusClient(collection_name=collection_name, ip=self.ip, port=self.port) for i in range(concurrent_num)]
+                            connections = [MilvusClient(collection_name=collection_name, host=self.hos, port=self.port) for i in range(concurrent_num)]
                             with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor:
                                 future_results = {executor.submit(
                                     self.do_query_qps, connections[index], query_vectors, top_k, search_param=search_param) : index for index in range(concurrent_num)}
@@ -207,7 +331,7 @@ class LocalRunner(Runner):
         elif run_type == "ann_accuracy":
             hdf5_source_file = collection["source_file"]
             collection_name = collection["collection_name"]
-            index_file_sizes = collection["index_file_sizes"]
+            segment_row_limits = collection["segment_row_limits"]
             index_types = collection["index_types"]
             index_params = collection["index_params"]
             top_ks = collection["top_ks"]
@@ -217,102 +341,179 @@ class LocalRunner(Runner):
             search_params = self.generate_combinations(search_params)
             # mapping to index param list
             index_params = self.generate_combinations(index_params)
-
             data_type, dimension, metric_type = parser.parse_ann_collection_name(collection_name)
             dataset = utils.get_dataset(hdf5_source_file)
-            if milvus_instance.exists_collection(collection_name):
-                logger.info("Re-create collection: %s" % collection_name)
-                milvus_instance.delete()
-                time.sleep(DELETE_INTERVAL_TIME)
             true_ids = np.array(dataset["neighbors"])
-            for index_file_size in index_file_sizes:
-                milvus_instance.create_collection(collection_name, dimension, index_file_size, metric_type)
-                logger.info(milvus_instance.describe())
+            vector_type = self.get_vector_type_from_metric(metric_type)
+            vec_field_name = utils.get_default_field_name(vector_type)
+            real_metric_type = utils.metric_type_trans(metric_type)
+
+            for segment_row_limit in segment_row_limits:
+                # re-create collection
+                if milvus_instance.exists_collection(collection_name):
+                    milvus_instance.drop()
+                    time.sleep(DELETE_INTERVAL_TIME)
+                milvus_instance.create_collection(dimension, segment_row_limit, data_type=vector_type)
                 insert_vectors = self.normalize(metric_type, np.array(dataset["train"]))
-                logger.debug(len(insert_vectors))
-                # Insert batch once
+                if len(insert_vectors) != dataset["train"].shape[0]:
+                    raise Exception("Row count of insert vectors: %d is not equal to dataset size: %d" % (len(insert_vectors), dataset["train"].shape[0]))
+                logger.debug("The row count of entities to be inserted: %d" % len(insert_vectors))
+                # insert batch once
                 # milvus_instance.insert(insert_vectors)
                 loops = len(insert_vectors) // INSERT_INTERVAL + 1
                 for i in range(loops):
                     start = i*INSERT_INTERVAL
                     end = min((i+1)*INSERT_INTERVAL, len(insert_vectors))
-                    tmp_vectors = insert_vectors[start:end]
                     if start < end:
+                        tmp_vectors = insert_vectors[start:end]
+                        ids = [i for i in range(start, end)]
                         if not isinstance(tmp_vectors, list):
-                            milvus_instance.insert(tmp_vectors.tolist(), ids=[i for i in range(start, end)])
+                            entities = milvus_instance.generate_entities(tmp_vectors.tolist(), ids)
+                            res_ids = milvus_instance.insert(entities, ids=ids)
                         else:
-                            milvus_instance.insert(tmp_vectors, ids=[i for i in range(start, end)])
-                    milvus_instance.flush()
-                logger.info("Table: %s, row count: %s" % (collection_name, milvus_instance.count()))
-                if milvus_instance.count() != len(insert_vectors):
-                    logger.error("Table row count is not equal to insert vectors")
-                    return
+                            entities = milvus_instance.generate_entities(tmp_vectors, ids)
+                            res_ids = milvus_instance.insert(entities, ids=ids)
+                        assert res_ids == ids
+                milvus_instance.flush()
+                res_count = milvus_instance.count()
+                logger.info("Table: %s, row count: %d" % (collection_name, res_count))
+                if res_count != len(insert_vectors):
+                    raise Exception("Table row count is not equal to insert vectors")
                 for index_type in index_types:
                     for index_param in index_params:
-                        logger.debug("Building index with param: %s" % json.dumps(index_param))
-                        milvus_instance.create_index(index_type, index_param=index_param)
-                        logger.info(milvus_instance.describe_index())
+                        logger.debug("Building index with param: %s, metric_type: %s" % (json.dumps(index_param), metric_type))
+                        milvus_instance.create_index(vec_field_name, index_type, metric_type, index_param=index_param)
                         logger.info("Start preload collection: %s" % collection_name)
                         milvus_instance.preload_collection()
                         for search_param in search_params:
                             for nq in nqs:
                                 query_vectors = self.normalize(metric_type, np.array(dataset["test"][:nq]))
+                                if not isinstance(query_vectors, list):
+                                    query_vectors = query_vectors.tolist()
                                 for top_k in top_ks:
-                                    logger.debug("Search nq: %d, top-k: %d, search_param: %s" % (nq, top_k, json.dumps(search_param)))
-                                    if not isinstance(query_vectors, list):
-                                        result = milvus_instance.query(query_vectors.tolist(), top_k, search_param=search_param)
-                                    else:
-                                        result = milvus_instance.query(query_vectors, top_k, search_param=search_param)
-                                    result_ids = result.id_array
+                                    logger.debug("Search nq: %d, top-k: %d, search_param: %s, metric_type: %s" % (nq, top_k, json.dumps(search_param), metric_type))
+                                    vector_query = {"vector": {vec_field_name: {
+                                        "topk": top_k,
+                                        "query": query_vectors,
+                                        "metric_type": real_metric_type,
+                                        "params": search_param}
+                                    }}
+                                    result = milvus_instance.query(vector_query)
+                                    result_ids = milvus_instance.get_ids(result)
+                                    # pdb.set_trace()
                                     acc_value = self.get_recall_value(true_ids[:nq, :top_k].tolist(), result_ids)
                                     logger.info("Query ann_accuracy: %s" % acc_value)
-
-
-        elif run_type == "stability":
-            (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
+        elif run_type == "accuracy":
+            (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name)
             search_params = collection["search_params"]
-            insert_xb = collection["insert_xb"]
-            insert_interval = collection["insert_interval"]
-            delete_xb = collection["delete_xb"]
-            # flush_interval = collection["flush_interval"]
-            # compact_interval = collection["compact_interval"]
-            during_time = collection["during_time"]
+            # mapping to search param list
+            search_params = self.generate_combinations(search_params)
+
+            top_ks = collection["top_ks"]
+            nqs = collection["nqs"]
+            collection_info = {
+                "dimension": dimension,
+                "metric_type": metric_type,
+                "dataset_name": collection_name,
+                "segment_row_limit": segment_row_limit
+            }
             if not milvus_instance.exists_collection():
-                logger.error(milvus_instance.show_collections())
                 logger.error("Table name: %s not existed" % collection_name)
                 return
-            g_top_k = int(collection["top_ks"].split("-")[1])
-            g_nq = int(collection["nqs"].split("-")[1])
-            l_top_k = int(collection["top_ks"].split("-")[0])
-            l_nq = int(collection["nqs"].split("-")[0])
+            logger.info(milvus_instance.count())
+            index_info = milvus_instance.describe_index()
+            logger.info(index_info)
+            milvus_instance.preload_collection()
+            true_ids_all = self.get_groundtruth_ids(collection_size)
+            vector_type = self.get_vector_type(data_type)
+            vec_field_name = utils.get_default_field_name(vector_type)
+            for search_param in search_params:
+                headers = ["Nq/Top-k"]
+                res = []
+                for nq in nqs:
+                    tmp_res = []
+                    for top_k in top_ks:
+                        search_param_group = {
+                            "nq": nq,
+                            "topk": top_k,
+                            "search_param": search_param,
+                            "metric_type": metric_type
+                        }
+                        logger.info("Query params: %s" % json.dumps(search_param_group))
+                        result_ids = self.do_query_ids(milvus_instance, collection_name, vec_field_name, top_k, nq, search_param=search_param)
+                        mem_used = milvus_instance.get_mem_info()["memory_used"]
+                        acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids)
+                        logger.info("Query accuracy: %s" % acc_value)
+                        tmp_res.append(acc_value)
+                        logger.info("Memory usage: %s" % mem_used)
+                    res.append(tmp_res)
+                headers.extend([str(top_k) for top_k in top_ks])
+                logger.info("Search param: %s" % json.dumps(search_param))
+                utils.print_table(headers, nqs, res)
+
+        elif run_type == "stability":
+            (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(collection_name)
+            during_time = collection["during_time"]
+            operations = collection["operations"]
+            if not milvus_instance.exists_collection():
+                logger.error(milvus_instance.show_collections())
+                raise Exception("Table name: %s not existed" % collection_name)
             milvus_instance.preload_collection()
             start_mem_usage = milvus_instance.get_mem_info()["memory_used"]
             start_row_count = milvus_instance.count()
-            logger.debug(milvus_instance.describe_index())
             logger.info(start_row_count)
-            start_time = time.time()
-            i = 0
-            ids = []
-            insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)]
+            vector_type = self.get_vector_type(data_type)
+            vec_field_name = utils.get_default_field_name(vector_type)
+            real_metric_type = utils.metric_type_trans(metric_type)
             query_vectors = [[random.random() for _ in range(dimension)] for _ in range(10000)]
+            if "insert" in operations:
+                insert_xb = operations["insert"]["xb"]
+            if "delete" in operations:
+                delete_xb = operations["delete"]["xb"]
+            if "query" in operations:
+                g_top_k = int(operations["query"]["top_ks"].split("-")[1])
+                l_top_k = int(operations["query"]["top_ks"].split("-")[0])
+                g_nq = int(operations["query"]["nqs"].split("-")[1])
+                l_nq = int(operations["query"]["nqs"].split("-")[0])
+                search_params = operations["query"]["search_params"]
+            i = 0
+            start_time = time.time()
             while time.time() < start_time + during_time * 60:
                 i = i + 1
-                for j in range(insert_interval):
-                    top_k = random.randint(l_top_k, g_top_k)
-                    nq = random.randint(l_nq, g_nq)
-                    search_param = {}
-                    for k, v in search_params.items():
-                        search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1]))
-                    logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param)))
-                    result = milvus_instance.query(query_vectors[0:nq], top_k, search_param=search_param)
-                count = milvus_instance.count()
-                insert_ids = [(count+x) for x in range(len(insert_vectors))]
-                ids.extend(insert_ids)
-                status, res = milvus_instance.insert(insert_vectors, ids=insert_ids)
-                logger.debug("%d, row_count: %d" % (i, milvus_instance.count()))
-                milvus_instance.delete_vectors(ids[-delete_xb:])
-                milvus_instance.flush()
-                milvus_instance.compact()
+                q = self.gen_executors(operations)
+                for name in q:
+                    try:
+                        if name == "insert":
+                            insert_ids = random.sample(list(range(collection_size)), insert_xb)
+                            insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)]
+                            entities = milvus_instance.generate_entities(insert_vectors, insert_ids)
+                            milvus_instance.insert(entities, ids=insert_ids)
+                        elif name == "delete":
+                            delete_ids = random.sample(list(range(collection_size)), delete_xb)
+                            milvus_instance.delete(delete_ids)
+                        elif name == "query":
+                            top_k = random.randint(l_top_k, g_top_k)
+                            nq = random.randint(l_nq, g_nq)
+                            search_param = {}
+                            for k, v in search_params.items():
+                                search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1]))
+                            logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param)))
+                            vector_query = {"vector": {vec_field_name: {
+                                "topk": top_k,
+                                "query": query_vectors[:nq],
+                                "metric_type": real_metric_type,
+                                "params": search_param}
+                            }}
+                            result = milvus_instance.query(vector_query)
+                        elif name in ["flush", "compact"]:
+                            func = getattr(milvus_instance, name)
+                            func()
+                        logger.debug(milvus_instance.count())
+                    except Exception as e:
+                        logger.error(name)
+                        logger.error(str(e))
+                        raise
+                logger.debug("Loop time: %d" % i)
             end_mem_usage = milvus_instance.get_mem_info()["memory_used"]
             end_row_count = milvus_instance.count()
             metrics = {
@@ -324,7 +525,233 @@ class LocalRunner(Runner):
             }
             logger.info(metrics)
 
+        elif run_type == "loop_stability":
+            # init data
+            milvus_instance.clean_db()
+            pull_interval = collection["pull_interval"]
+            collection_num = collection["collection_num"]
+            concurrent = collection["concurrent"] if "concurrent" in collection else False
+            concurrent_num = collection_num
+            dimension = collection["dimension"] if "dimension" in collection else 128
+            insert_xb = collection["insert_xb"] if "insert_xb" in collection else 100000
+            index_types = collection["index_types"] if "index_types" in collection else ['ivf_sq8']
+            index_param = {"nlist": 256}
+            collection_names = []
+            milvus_instances_map = {}
+            insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)]
+            ids = [i for i in range(insert_xb)]
+            # initialize and prepare
+            for i in range(collection_num):
+                name = utils.get_unique_name(prefix="collection_%d_" % i)
+                collection_names.append(name)
+                metric_type = random.choice(["l2", "ip"])
+                segment_row_limit = random.randint(50000, 100000)
+                # default float_vector
+                milvus_instance = MilvusClient(collection_name=name, host=self.host)
+                milvus_instance.create_collection(dimension, segment_row_limit, other_fields=None)
+                index_type = random.choice(index_types)
+                field_name = utils.get_default_field_name()
+                milvus_instance.create_index(field_name, index_type, metric_type, index_param=index_param)
+                logger.info(milvus_instance.describe_index())
+                insert_vectors = utils.normalize(metric_type, insert_vectors)
+                entities = milvus_instance.generate_entities(insert_vectors, ids)
+                res_ids = milvus_instance.insert(entities, ids=ids)
+                milvus_instance.flush()
+                milvus_instances_map.update({name: milvus_instance})
+                logger.info(milvus_instance.describe_index())
+
+            # loop time unit: min -> s
+            pull_interval_seconds = pull_interval * 60
+            tasks = ["insert_rand", "delete_rand", "query_rand", "flush", "compact"]
+            i = 1
+            while True:
+                logger.info("Loop time: %d" % i)
+                start_time = time.time()
+                while time.time() - start_time < pull_interval_seconds:
+                    if concurrent:
+                        threads = []
+                        for name in collection_names:
+                            task_name = random.choice(tasks)
+                            task_run = getattr(milvus_instances_map[name], task_name)
+                            t = threading.Thread(target=task_run, args=())
+                            threads.append(t)
+                            t.start()
+                        for t in threads:
+                            t.join()
+                        # with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor:
+                        #     future_results = {executor.submit(getattr(milvus_instances_map[mp[j][0]], mp[j][1])): j for j in range(concurrent_num)}
+                        #     for future in concurrent.futures.as_completed(future_results):
+                        #         future.result()
+                    else:
+                        tmp_collection_name = random.choice(collection_names)
+                        task_name = random.choice(tasks)
+                        logger.info(tmp_collection_name)
+                        logger.info(task_name)
+                        task_run = getattr(milvus_instances_map[tmp_collection_name], task_name)
+                        task_run()
+                # new connection
+                # for name in collection_names:
+                #     milvus_instance = MilvusClient(collection_name=name, host=self.host)
+                #     milvus_instances_map.update({name: milvus_instance})
+                i = i + 1
+
+        elif run_type == "locust_mix_performance":
+            (data_type, collection_size, segment_row_limit, dimension, metric_type) = parser.collection_parser(
+                collection_name)
+            ni_per = collection["ni_per"]
+            build_index = collection["build_index"]
+            vector_type = self.get_vector_type(data_type)
+            index_field_name = utils.get_default_field_name(vector_type)
+            # drop exists collection
+            if milvus_instance.exists_collection():
+                milvus_instance.drop()
+                time.sleep(10)
+            # create collection
+            other_fields = collection["other_fields"] if "other_fields" in collection else None
+            milvus_instance.create_collection(dimension, segment_row_limit, data_type=DataType.FLOAT_VECTOR, collection_name=collection_name, other_fields=other_fields)
+            logger.info(milvus_instance.get_info())
+            # insert entities
+            insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(ni_per)]
+            insert_ids = random.sample(list(range(collection_size)), ni_per)
+            insert_vectors = utils.normalize(metric_type, insert_vectors)
+            entities = milvus_instance.generate_entities(insert_vectors, insert_ids, collection_name)
+            milvus_instance.insert(entities, ids=insert_ids)
+            # flush
+            milvus_instance.flush()
+            logger.info(milvus_instance.get_stats())
+            logger.debug("Table row counts: %d" % milvus_instance.count())
+            # create index
+            if build_index is True:
+                index_type = collection["index_type"]
+                index_param = collection["index_param"]
+                logger.debug("Start build index for last file")
+                milvus_instance.create_index(index_field_name, index_type, metric_type, index_param)
+                logger.debug(milvus_instance.describe_index())
+            # locust
+            task = collection["tasks"]
+            task_file = utils.get_unique_name()
+            task_file_script = task_file + '.py'
+            task_file_csv = task_file + '_stats.csv'
+            task_types = task["types"]
+            connection_type = "single"
+            connection_num = task["connection_num"]
+            if connection_num > 1:
+                connection_type = "multi"
+            clients_num = task["clients_num"]
+            hatch_rate = task["hatch_rate"]
+            during_time = task["during_time"]
+            def_strs = ""
+            # define def str
+            for task_type in task_types:
+                type = task_type["type"]
+                weight = task_type["weight"]
+                if type == "flush":
+                    def_str = """
+    @task(%d)
+    def flush(self):
+        client = get_client(collection_name)
+        client.flush(collection_name=collection_name)
+                        """ % weight
+                if type == "compact":
+                    def_str = """
+    @task(%d)
+    def compact(self):
+        client = get_client(collection_name)
+        client.compact(collection_name)
+                        """ % weight
+                if type == "query":
+                    def_str = """
+    @task(%d)
+    def query(self):
+        client = get_client(collection_name)
+        params = %s
+        X = [[random.random() for i in range(dim)] for i in range(params["nq"])]
+        vector_query = {"vector": {"%s": {
+        "topk": params["top_k"], 
+        "query": X, 
+        "metric_type": "%s", 
+        "params": params["search_param"]}}}
+        client.query(vector_query, filter_query=params["filters"], collection_name=collection_name)
+                        """ % (weight, task_type["params"], index_field_name, utils.metric_type_trans(metric_type))
+                if type == "insert":
+                    def_str = """
+    @task(%d)
+    def insert(self):
+        client = get_client(collection_name)
+        params = %s
+        insert_ids = random.sample(list(range(100000)), params["nb"])
+        insert_vectors = [[random.random() for _ in range(dim)] for _ in range(params["nb"])]
+        insert_vectors = utils.normalize("l2", insert_vectors)
+        entities = generate_entities(insert_vectors, insert_ids)
+        client.insert(entities,ids=insert_ids, collection_name=collection_name)
+                    """ % (weight, task_type["params"])
+                if type == "delete":
+                    def_str = """
+    @task(%d)
+    def delete(self):
+        client = get_client(collection_name)
+        ids = [random.randint(1, 1000000) for i in range(1)]
+        client.delete(ids, collection_name)
+                        """ % weight
+                def_strs += def_str
+                print(def_strs)
+                # define locust code str
+                code_str = """
+import random
+import json
+from locust import User, task, between
+from locust_task import MilvusTask
+from client import MilvusClient
+import utils
+
+host = '%s'
+port = %s
+collection_name = '%s'
+dim = %s
+connection_type = '%s'
+m = MilvusClient(host=host, port=port)
+
+
+def get_client(collection_name):
+    if connection_type == 'single':
+        return MilvusTask(m=m)
+    elif connection_type == 'multi':
+        return MilvusTask(connection_type='multi', host=host, port=port, collection_name=collection_name)
+  
+        
+def generate_entities(vectors, ids):
+    return m.generate_entities(vectors, ids, collection_name)
+
+
+class MixTask(User):
+    wait_time = between(0.001, 0.002)
+    %s
+        """ % (self.host, self.port, collection_name, dimension, connection_type, def_strs)
+            with open(task_file_script, "w+") as fd:
+                fd.write(code_str)
+            locust_cmd = "locust -f %s --headless --csv=%s -u %d -r %d -t %s" % (
+                task_file_script,
+                task_file,
+                clients_num,
+                hatch_rate,
+                during_time)
+            logger.info(locust_cmd)
+            try:
+                res = os.system(locust_cmd)
+            except Exception as e:
+                logger.error(str(e))
+                return
+
+            # . retrieve and collect test statistics
+            metric = None
+            with open(task_file_csv, newline='') as fd:
+                dr = csv.DictReader(fd)
+                for row in dr:
+                    if row["Name"] != "Aggregated":
+                        continue
+                    metric = row
+            logger.info(metric)
+
         else:
-            logger.warning("Run type not defined")
-            return
-        logger.debug("Test finished") 
+            raise Exception("Run type not defined")
+        logger.debug("All test finished")

tests/milvus_benchmark/locust_file.py

@@ -0,0 +1,30 @@
+
+import random
+from locust import HttpUser, task, between
+
+
+collection_name = "random_1m_2048_512_ip_sq8"
+headers = {'Content-Type': "application/json"}
+url = '/collections/%s/vectors' % collection_name
+top_k = 2
+nq = 1
+dim = 512
+vectors =  [[random.random() for _ in range(dim)] for _ in range(nq)] 
+data = {
+    "search":{
+        "topk": top_k,
+        "vectors": vectors,
+        "params": {
+            "nprobe": 1
+        }
+    }
+}
+
+class MyUser(HttpUser):
+    wait_time = between(0, 0.1)
+    host = "http://192.168.1.112:19122"
+
+    @task
+    def search(self):
+        response = self.client.put(url=url, json=data, headers=headers, timeout=2)
+        print(response) 

tests/milvus_benchmark/locust_flush_task.py

@@ -0,0 +1,34 @@
+import random
+from locust import User, task, between
+from locust_task import MilvusTask
+from client import MilvusClient
+from milvus import DataType
+
+connection_type = "single"
+host = "192.168.1.6"
+port = 19530
+collection_name = "create_collection_CZkkwJgo"
+dim = 128
+nb = 50000
+segment_row_limit = 10000
+m = MilvusClient(host=host, port=port, collection_name=collection_name)
+m.clean_db()
+m.create_collection(dim, segment_row_limit, data_type=DataType.FLOAT_VECTOR, auto_id=True, other_fields=None)
+vectors = [[random.random() for _ in range(dim)] for _ in range(nb)]
+entities = m.generate_entities(vectors)
+
+
+class FlushTask(User):
+    wait_time = between(0.001, 0.002)
+    if connection_type == "single":
+        client = MilvusTask(m=m)
+    else:
+        client = MilvusTask(host=host, port=port, collection_name=collection_name)
+
+    # def insert(self):
+    #     self.client.insert(entities)
+
+    @task(1)
+    def flush(self):
+        self.client.insert(entities)
+        self.client.flush(collection_name)

tests/milvus_benchmark/locust_get_entity_task.py

@@ -0,0 +1,37 @@
+import logging
+import random
+from locust import User, task, between
+from locust_task import MilvusTask
+from client import MilvusClient
+from milvus import DataType
+
+connection_type = "single"
+host = "192.168.1.6"
+port = 19530
+collection_name = "sift_10m_100000_128_l2"
+dim = 128
+segment_row_limit = 10000
+m = MilvusClient(host=host, port=port, collection_name=collection_name)
+# m.clean_db()
+# m.create_collection(dim, segment_row_limit, data_type=DataType.FLOAT_VECTOR, auto_id=True, other_fields=None)
+nb = 6000
+# vectors = [[random.random() for _ in range(dim)] for _ in range(nb)]
+# entities = m.generate_entities(vectors)
+ids = [i for i in range(nb)]
+
+class GetEntityTask(User):
+    wait_time = between(0.001, 0.002)
+    if connection_type == "single":
+        client = MilvusTask(m=m)
+    else:
+        client = MilvusTask(host=host, port=port, collection_name=collection_name)
+
+    # def insert(self):
+    #     self.client.insert(entities)
+
+    @task(1)
+    def get_entity_by_id(self):
+        # num = random.randint(100, 200)
+        # get_ids = random.sample(ids, num)
+        self.client.get_entities([0])
+        # logging.getLogger().info(len(get_res))

tests/milvus_benchmark/locust_insert_task.py

@@ -0,0 +1,35 @@
+import random
+from locust import User, task, between
+from locust_task import MilvusTask
+from client import MilvusClient
+from milvus import DataType
+
+connection_type = "single"
+host = "192.168.1.6"
+port = 19530
+collection_name = "create_collection_hello"
+dim = 128
+nb = 50000
+segment_row_limit = 10000
+m = MilvusClient(host=host, port=port, collection_name=collection_name)
+# m.clean_db()
+m.create_collection(dim, segment_row_limit, data_type=DataType.FLOAT_VECTOR, auto_id=True, other_fields=None)
+vectors = [[random.random() for _ in range(dim)] for _ in range(nb)]
+entities = m.generate_entities(vectors)
+
+
+class FlushTask(User):
+    wait_time = between(0.001, 0.002)
+    if connection_type == "single":
+        client = MilvusTask(m=m)
+    else:
+        client = MilvusTask(host=host, port=port, collection_name=collection_name)
+
+    @task(1)
+    def insert(self):
+        self.client.insert(entities)
+    # @task(1)
+    # def create_partition(self):
+    #     tag = 'tag_'.join(random.choice(string.ascii_letters) for _ in range(8))
+    #     logging.info(tag)
+    #     self.client.create_partition(tag, collection_name)

tests/milvus_benchmark/locust_search_task.py

@@ -0,0 +1,38 @@
+import logging
+import random
+from locust import User, task, between
+from locust_task import MilvusTask
+from client import MilvusClient
+
+connection_type = "single"
+host = "192.168.1.6"
+port = 19530
+collection_name = "sift_1m_2000000_128_l2_2"
+m = MilvusClient(host=host, port=port, collection_name=collection_name)
+dim = 128
+top_k = 10
+nq = 1
+X = [[random.random() for i in range(dim)] for i in range(nq)]
+search_params = {"nprobe": 32}
+vector_query = {"vector": {'float_vector': {
+    "topk": top_k,
+    "query": X,
+    "params": search_params,
+    'metric_type': 'L2'}}}
+# m.clean_db()
+
+
+class QueryTask(User):
+    wait_time = between(0.001, 0.002)
+    if connection_type == "single":
+        client = MilvusTask(m=m)
+    else:
+        client = MilvusTask(host=host, port=port, collection_name=collection_name)
+    # client = MilvusClient(collection_name, host, port)
+
+    def preload(self):
+        self.client.preload_collection()
+
+    @task(10)
+    def query(self):
+        self.client.query(vector_query, collection_name=collection_name)

tests/milvus_benchmark/locust_task.py

@@ -0,0 +1,35 @@
+import time
+import random
+import logging
+from locust import User, events
+from client import MilvusClient
+
+
+class MilvusTask(object):
+    def __init__(self, *args, **kwargs):
+        self.request_type = "grpc"
+        connection_type = kwargs.get("connection_type")
+        if connection_type == "single":
+            self.m = kwargs.get("m")
+        elif connection_type == "multi":
+            host = kwargs.get("host")
+            port = kwargs.get("port")
+            collection_name = kwargs.get("collection_name")
+            self.m = MilvusClient(host=host, port=port, collection_name=collection_name)
+
+    def __getattr__(self, name):
+        func = getattr(self.m, name)
+
+        def wrapper(*args, **kwargs):
+            start_time = time.time()
+            try:
+                result = func(*args, **kwargs)
+                total_time = int((time.time() - start_time) * 1000)
+                events.request_success.fire(request_type=self.request_type, name=name, response_time=total_time,
+                                            response_length=0)
+            except Exception as e:
+                total_time = int((time.time() - start_time) * 1000)
+                events.request_failure.fire(request_type=self.request_type, name=name, response_time=total_time,
+                                            exception=e, response_length=0)
+
+        return wrapper

tests/milvus_benchmark/locust_tasks.py

@@ -0,0 +1,41 @@
+import random
+import time
+import logging
+from locust import TaskSet, task
+
+dim = 128
+
+
+class Tasks(TaskSet):
+
+    @task
+    def query(self):
+        top_k = 10
+        search_param = {"nprobe": 16}
+        X = [[random.random() for i in range(dim)]]
+        vector_query = {"vector": {"float_vector": {
+            "topk": top_k, 
+            "query": X, 
+            "metric_type": "L2", 
+            "params": search_param}
+        }}
+        filter_query = None
+        self.client.query(vector_query, filter_query=filter_query)
+
+    @task
+    def flush(self):
+        self.client.flush()
+
+    @task
+    def compact(self):
+        self.client.compact()
+
+    @task
+    def delete(self):
+        self.client.delete([random.randint(1, 1000000)])
+
+    @task
+    def insert(self):
+        id = random.randint(10000000, 10000000000)
+        X = [[random.random() for i in range(dim)] for i in range(1)]
+        self.client.insert(X, ids=[id])

tests/milvus_benchmark/locust_user.py

@@ -0,0 +1,66 @@
+import logging
+import random
+import pdb
+import gevent
+import gevent.monkey
+gevent.monkey.patch_all()
+
+from locust import User, between, events, stats
+from locust.env import Environment
+from locust.stats import stats_printer, print_stats
+from locust.log import setup_logging, greenlet_exception_logger
+
+from locust_tasks import Tasks
+from client import MilvusClient
+from locust_task import MilvusTask
+
+
+class MyUser(User):
+    # task_set = None
+    wait_time = between(0.001, 0.002)
+    
+
+def locust_executor(host, port, collection_name, connection_type="single", run_params=None):
+    m = MilvusClient(host=host, port=port, collection_name=collection_name)
+    MyUser.tasks = {}
+    tasks = run_params["tasks"]
+    for op, weight in tasks.items():
+        task = {eval("Tasks."+op): weight}
+        MyUser.tasks.update(task)
+    # MyUser.tasks = {Tasks.query: 1, Tasks.flush: 1}
+    MyUser.client = MilvusTask(host=host, port=port, collection_name=collection_name, connection_type=connection_type, m=m)
+    env = Environment(events=events, user_classes=[MyUser])
+    runner = env.create_local_runner()
+    # setup logging
+    logger = logging.getLogger("__locust__")
+    setup_logging("INFO", None)
+    greenlet_exception_logger(logger=logger)
+    gevent.spawn(stats_printer(env.stats))
+    # env.create_web_ui("127.0.0.1", 8089)
+    # gevent.spawn(stats_printer(env.stats), env, "test", full_history=True)
+    # events.init.fire(environment=env, runner=runner)
+    clients_num = run_params["clients_num"]
+    spawn_rate = run_params["spawn_rate"]
+    during_time = run_params["during_time"]
+    runner.start(clients_num, spawn_rate=spawn_rate)
+    gevent.spawn_later(during_time, lambda: runner.quit())
+    runner.greenlet.join()
+    print_stats(env.stats)
+    result = {
+        "rps": round(env.stats.total.current_rps, 1),
+        "fail_ratio": env.stats.total.fail_ratio,
+        "max_response_time": round(env.stats.total.max_response_time, 1),
+        "min_response_time": round(env.stats.total.avg_response_time, 1)
+    }
+    logging.info(result)
+    runner.stop()
+    return result
+
+
+if __name__ == '__main__':
+    connection_type = "single"
+    host = "192.168.1.112"
+    port = 19530
+    collection_name = "sift_1m_2000000_128_l2_2"
+    run_params = {"tasks": {"query": 1, "flush": 1}, "clients_num": 1, "spawn_rate": 1, "during_time": 3}
+    locust_executor(host, port, collection_name, run_params=run_params)

tests/milvus_benchmark/main.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import time
+from datetime import datetime
 import pdb
 import argparse
 import logging
@@ -17,12 +18,23 @@ import parser
 DEFAULT_IMAGE = "milvusdb/milvus:latest"
 LOG_FOLDER = "logs"
 NAMESPACE = "milvus"
+LOG_PATH = "/test/milvus/benchmark/logs/"
 
-logging.basicConfig(format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
-    datefmt='%Y-%m-%d:%H:%M:%S',
-    level=logging.DEBUG)
-logger = logging.getLogger("milvus_benchmark")
-
+logger = logging.getLogger('milvus_benchmark')
+logger.setLevel(logging.DEBUG)
+# create file handler which logs even debug messages
+fh = logging.FileHandler(LOG_PATH+'benchmark-{:%Y-%m-%d}.log'.format(datetime.now()))
+fh.setLevel(logging.DEBUG)
+# create console handler with a higher log level
+ch = logging.StreamHandler()
+ch.setLevel(logging.INFO)
+# create formatter and add it to the handlers
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+fh.setFormatter(formatter)
+ch.setFormatter(formatter)
+# add the handlers to the logger
+logger.addHandler(fh)
+logger.addHandler(ch)
 
 def positive_int(s):
     i = None
@@ -38,7 +50,7 @@ def positive_int(s):
 def get_image_tag(image_version, image_type):
     return "%s-%s-centos7-release" % (image_version, image_type)
     # return "%s-%s-centos7-release" % ("0.7.1", image_type)
-    # return "%s-%s-centos7-release" % ("PR-2159", image_type)
+    # return "%s-%s-centos7-release" % ("PR-2780", image_type)
 
 
 def queue_worker(queue):
@@ -46,6 +58,7 @@ def queue_worker(queue):
         q = queue.get()
         suite = q["suite"]
         server_host = q["server_host"]
+        deploy_mode = q["deploy_mode"]
         image_type = q["image_type"]
         image_tag = q["image_tag"]
 
@@ -58,10 +71,12 @@ def queue_worker(queue):
         collections = run_params["collections"]
         for collection in collections:
             # run tests
-            server_config = collection["server"]
+            milvus_config = collection["milvus"] if "milvus" in collection else None
+            server_config = collection["server"] if "server" in collection else None
+            logger.debug(milvus_config)
             logger.debug(server_config)
             runner = K8sRunner()
-            if runner.init_env(server_config, server_host, image_type, image_tag):
+            if runner.init_env(milvus_config, server_config, server_host, deploy_mode, image_type, image_tag):
                 logger.debug("Start run tests")
                 try:
                     runner.run(run_type, collection)
@@ -69,10 +84,12 @@ def queue_worker(queue):
                     logger.error(str(e))
                     logger.error(traceback.format_exc())
                 finally:
+                    time.sleep(60)
                     runner.clean_up()
             else:
                 logger.error("Runner init failed")
-    logger.debug("All task finished in queue: %s" % server_host)
+    if server_host:
+        logger.debug("All task finished in queue: %s" % server_host)
 
 
 def main():
@@ -88,6 +105,10 @@ def main():
         metavar='FILE',
         default='',
         help="load test schedule from FILE")
+    arg_parser.add_argument(
+        "--deploy-mode",
+        default='',
+        help="single or shards")
 
     # local mode
     arg_parser.add_argument(
@@ -116,15 +137,17 @@ def main():
         if not args.image_version:
             raise Exception("Image version not given")
         image_version = args.image_version
+        deploy_mode = args.deploy_mode
         with open(args.schedule_conf) as f:
             schedule_config = full_load(f)
             f.close()
         queues = []
-        server_names = set()
+        # server_names = set()
+        server_names = []
         for item in schedule_config:
-            server_host = item["server"]
+            server_host = item["server"] if "server" in item else ""
             suite_params = item["suite_params"]
-            server_names.add(server_host)
+            server_names.append(server_host)
             q = Queue()
             for suite_param in suite_params:
                 suite = "suites/"+suite_param["suite"]
@@ -133,11 +156,12 @@ def main():
                 q.put({
                     "suite": suite,
                     "server_host": server_host,
+                    "deploy_mode": deploy_mode,
                     "image_tag": image_tag,
                     "image_type": image_type
                 })
             queues.append(q)
-        logger.debug(server_names)
+        logging.error(queues)
         thread_num = len(server_names)
         processes = []
 
@@ -145,10 +169,12 @@ def main():
             x = Process(target=queue_worker, args=(queues[i], ))
             processes.append(x)
             x.start()
-            time.sleep(5)
+            time.sleep(10)
         for x in processes:
             x.join()
 
+        # queue_worker(queues[0])
+
     elif args.local:
         # for local mode
         host = args.host

tests/milvus_benchmark/mix_task.py

@@ -0,0 +1,45 @@
+import random
+from locust import User, task, between
+from locust_task import MilvusTask
+from client import MilvusClient
+
+connection_type = "single"
+host = "192.168.1.29"
+port = 19530
+collection_name = "sift_128_euclidean"
+dim = 128
+m = MilvusClient(host=host, port=port, collection_name=collection_name)
+
+
+class MixTask(User):
+    wait_time = between(0.001, 0.002)
+    print("in query task")
+    if connection_type == "single":
+        client = MilvusTask(m=m)
+    else:
+        client = MilvusTask(host=host, port=port, collection_name=collection_name)
+
+    @task(30)
+    def query(self):
+        top_k = 10
+        X = [[random.random() for i in range(dim)] for i in range(1)]
+        search_param = {"nprobe": 16}
+        self.client.query(X, top_k, search_param)
+
+    @task(10)
+    def insert(self):
+        id = random.randint(10000000, 10000000000)
+        X = [[random.random() for i in range(dim)] for i in range(1)]
+        self.client.insert(X, ids=[id])
+
+    @task(1)
+    def flush(self):
+        self.client.flush()
+
+    @task(5)
+    def delete(self):
+        self.client.delete([random.randint(1, 1000000)])
+
+    @task(1)
+    def compact(self):
+        self.client.compact()

tests/milvus_benchmark/parser.py

@@ -32,7 +32,7 @@ def collection_parser(collection_name):
 def parse_ann_collection_name(collection_name):
     data_type = collection_name.split("_")[0]
     dimension = int(collection_name.split("_")[1])
-    metric = collection_name.split("_")[-1]
+    metric = collection_name.split("_")[2]
     # metric = collection_name.attrs['distance']
     # dimension = len(collection_name["train"][0])
     if metric == "euclidean":
@@ -43,7 +43,7 @@ def parse_ann_collection_name(collection_name):
         metric_type = "jaccard"
     elif metric == "hamming":
         metric_type = "hamming"
-    return ("ann_"+data_type, dimension, metric_type)
+    return (data_type, dimension, metric_type)
 
 
 def search_params_parser(param):
@@ -83,4 +83,4 @@ def search_params_parser(param):
     else:
         logger.warning("Invalid format nprobes: %s" % str(nprobes))    
 
-    return top_ks, nqs, nprobes
+    return top_ks, nqs, nprobes

tests/milvus_benchmark/report.py

@@ -1,10 +0,0 @@
-# from tablereport import Table
-# from tablereport.shortcut import write_to_excel
-
-# RESULT_FOLDER = "results"
-
-# def create_table(headers, bodys, table_name):
-# 	table = Table(header=[headers],
-# 	              body=[bodys])
-
-# 	write_to_excel('%s/%s.xlsx' % (RESULT_FOLDER, table_name), table)

tests/milvus_benchmark/requirements.txt

@@ -1,4 +1,4 @@
-pymilvus-test>=0.2.0
+pymilvus-test==0.4.16
 scipy==1.3.1
 scikit-learn==0.19.1
 h5py==2.7.1
@@ -6,6 +6,7 @@ h5py==2.7.1
 pyyaml>=5.1
 tableprint==0.8.0
 ansicolors==1.1.8
-scipy==1.3.1
 kubernetes==10.0.1
 # rq==1.2.0
+locust>=1.0.2
+pymongo==3.10.0

tests/milvus_benchmark/results/__init__.py

@@ -0,0 +1,11 @@
+
+class Reporter(object):
+    def __init__(self):
+        pass
+
+    def report(self, result):
+        pass
+    
+
+class BaseResult(object):
+    pass

tests/milvus_benchmark/runner.py

@@ -1,12 +1,15 @@
 import os
+import threading
 import logging
 import pdb
 import time
 import random
+import grpc
 from multiprocessing import Process
 from itertools import product
 import numpy as np
 import sklearn.preprocessing
+from milvus import DataType
 from client import MilvusClient
 import utils
 import parser
@@ -15,7 +18,7 @@ logger = logging.getLogger("milvus_benchmark.runner")
 
 VECTORS_PER_FILE = 1000000
 SIFT_VECTORS_PER_FILE = 100000
-JACCARD_VECTORS_PER_FILE = 2000000
+BINARY_VECTORS_PER_FILE = 2000000
 
 MAX_NQ = 10001
 FILE_PREFIX = "binary_"
@@ -24,9 +27,7 @@ FILE_PREFIX = "binary_"
 SRC_BINARY_DATA_DIR = '/test/milvus/raw_data/random/'
 SIFT_SRC_DATA_DIR = '/test/milvus/raw_data/sift1b/'
 DEEP_SRC_DATA_DIR = '/test/milvus/raw_data/deep1b/'
-JACCARD_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/'
-HAMMING_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/'
-STRUCTURE_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/'
+BINARY_SRC_DATA_DIR = '/test/milvus/raw_data/binary/'
 SIFT_SRC_GROUNDTRUTH_DATA_DIR = SIFT_SRC_DATA_DIR + 'gnd'
 
 WARM_TOP_K = 1
@@ -57,12 +58,8 @@ def gen_file_name(idx, dimension, data_type):
         fname = SIFT_SRC_DATA_DIR+fname
     elif data_type == "deep":
         fname = DEEP_SRC_DATA_DIR+fname
-    elif data_type == "jaccard":
-        fname = JACCARD_SRC_DATA_DIR+fname
-    elif data_type == "hamming":
-        fname = HAMMING_SRC_DATA_DIR+fname
-    elif data_type == "sub" or data_type == "super":
-        fname = STRUCTURE_SRC_DATA_DIR+fname
+    elif data_type == "binary":
+        fname = BINARY_SRC_DATA_DIR+fname
     return fname
 
 
@@ -76,12 +73,8 @@ def get_vectors_from_binary(nq, dimension, data_type):
         file_name = SIFT_SRC_DATA_DIR+'query.npy'
     elif data_type == "deep":
         file_name = DEEP_SRC_DATA_DIR+'query.npy'
-    elif data_type == "jaccard":
-        file_name = JACCARD_SRC_DATA_DIR+'query.npy'
-    elif data_type == "hamming":
-        file_name = HAMMING_SRC_DATA_DIR+'query.npy'
-    elif data_type == "sub" or data_type == "super":
-        file_name = STRUCTURE_SRC_DATA_DIR+'query.npy'
+    elif data_type == "binary":
+        file_name = BINARY_SRC_DATA_DIR+'query.npy'
     data = np.load(file_name)
     vectors = data[0:nq].tolist()
     return vectors
@@ -91,6 +84,32 @@ class Runner(object):
     def __init__(self):
         pass
 
+    def gen_executors(self, operations):
+        l = []
+        for name, operation in operations.items():
+            weight = operation["weight"] if "weight" in operation else 1
+            l.extend([name] * weight)
+        random.shuffle(l)
+        return l
+        
+    def get_vector_type(self, data_type):
+        vector_type = ''
+        if data_type in ["random", "sift", "deep", "glove"]:
+            vector_type = DataType.FLOAT_VECTOR
+        elif data_type in ["binary"]:
+            vector_type = DataType.BINARY_VECTOR
+        else:
+            raise Exception("Data type: %s not defined" % data_type)
+        return vector_type
+
+    def get_vector_type_from_metric(self, metric_type):
+        vector_type = ''
+        if metric_type in ["hamming", "jaccard"]:
+            vector_type = DataType.BINARY_VECTOR
+        else:
+            vector_type = DataType.FLOAT_VECTOR
+        return vector_type
+
     def normalize(self, metric_type, X):
         if metric_type == "ip":
             logger.info("Set normalize for metric_type: %s" % metric_type)
@@ -100,7 +119,7 @@ class Runner(object):
             X = X.astype(np.float32)
         elif metric_type in ["jaccard", "hamming", "sub", "super"]:
             tmp = []
-            for index, item in enumerate(X):
+            for item in X:
                 new_vector = bytes(np.packbits(item, axis=-1).tolist())
                 tmp.append(new_vector)
             X = tmp
@@ -148,56 +167,111 @@ class Runner(object):
                 vectors_per_file = 10000
         elif data_type == "sift":
             vectors_per_file = SIFT_VECTORS_PER_FILE
-        elif data_type in ["jaccard", "hamming", "sub", "super"]:
-            vectors_per_file = JACCARD_VECTORS_PER_FILE
+        elif data_type in ["binary"]:
+            vectors_per_file = BINARY_VECTORS_PER_FILE
         else:
             raise Exception("data_type: %s not supported" % data_type)
-        if size % vectors_per_file or ni > vectors_per_file:
+        if size % vectors_per_file or size % ni:
             raise Exception("Not invalid collection size or ni")
-        file_num = size // vectors_per_file
-        for i in range(file_num):
-            file_name = gen_file_name(i, dimension, data_type)
-            # logger.info("Load npy file: %s start" % file_name)
-            data = np.load(file_name)
-            # logger.info("Load npy file: %s end" % file_name)
-            loops = vectors_per_file // ni
-            for j in range(loops):
-                vectors = data[j*ni:(j+1)*ni].tolist()
+        i = 0
+        while i < (size // vectors_per_file):
+            vectors = []
+            if vectors_per_file >= ni:
+                file_name = gen_file_name(i, dimension, data_type)
+                # logger.info("Load npy file: %s start" % file_name)
+                data = np.load(file_name)
+                # logger.info("Load npy file: %s end" % file_name)
+                for j in range(vectors_per_file // ni):
+                    vectors = data[j*ni:(j+1)*ni].tolist()
+                    if vectors:
+                        # start insert vectors
+                        start_id = i * vectors_per_file + j * ni
+                        end_id = start_id + len(vectors)
+                        logger.info("Start id: %s, end id: %s" % (start_id, end_id))
+                        ids = [k for k in range(start_id, end_id)]
+                        entities = milvus.generate_entities(vectors, ids)
+                        ni_start_time = time.time()
+                        try:
+                            res_ids = milvus.insert(entities, ids=ids)
+                        except grpc.RpcError as e:
+                            if e.code() == grpc.StatusCode.UNAVAILABLE:
+                                logger.debug("Retry insert")
+                                def retry():
+                                    res_ids = milvus.insert(entities, ids=ids)
+
+                                t0 = threading.Thread(target=retry)
+                                t0.start()
+                                t0.join()
+                                logger.debug("Retry successfully")
+                            raise e
+                        assert ids == res_ids
+                        # milvus.flush()
+                        logger.debug(milvus.count())
+                        ni_end_time = time.time()
+                        total_time = total_time + ni_end_time - ni_start_time
+                i += 1
+            else:
+                vectors.clear()
+                loops = ni // vectors_per_file
+                for j in range(loops):
+                    file_name = gen_file_name(loops*i+j, dimension, data_type)
+                    data = np.load(file_name)
+                    vectors.extend(data.tolist())
                 if vectors:
-                    ni_start_time = time.time()
-                    # start insert vectors
-                    start_id = i * vectors_per_file + j * ni
+                    start_id = i * vectors_per_file
                     end_id = start_id + len(vectors)
                     logger.info("Start id: %s, end id: %s" % (start_id, end_id))
                     ids = [k for k in range(start_id, end_id)]
-                    status, ids = milvus.insert(vectors, ids=ids)
+                    entities = milvus.generate_entities(vectors, ids)
+                    ni_start_time = time.time()
+                    try:
+                        res_ids = milvus.insert(entities, ids=ids)
+                    except grpc.RpcError as e:
+                        if e.code() == grpc.StatusCode.UNAVAILABLE:
+                            logger.debug("Retry insert")
+                            def retry():
+                                res_ids = milvus.insert(entities, ids=ids)
+
+                            t0 = threading.Thread(target=retry)
+                            t0.start()
+                            t0.join()
+                            logger.debug("Retry successfully")
+                        raise e
+
+                    assert ids == res_ids
                     # milvus.flush()
                     logger.debug(milvus.count())
                     ni_end_time = time.time()
                     total_time = total_time + ni_end_time - ni_start_time
-
+                i += loops
         qps = round(size / total_time, 2)
-        ni_time = round(total_time / (loops * file_num), 2)
+        ni_time = round(total_time / (size / ni), 2)
         bi_res["total_time"] = round(total_time, 2)
         bi_res["qps"] = qps
         bi_res["ni_time"] = ni_time
         return bi_res
 
-    def do_query(self, milvus, collection_name, top_ks, nqs, run_count=1, search_param=None):
+    def do_query(self, milvus, collection_name, vec_field_name, top_ks, nqs, run_count=1, search_param=None, filter_query=None):
         bi_res = []
         (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
         base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type)
         for nq in nqs:
             tmp_res = []
-            vectors = base_query_vectors[0:nq]
+            query_vectors = base_query_vectors[0:nq]
             for top_k in top_ks:
                 avg_query_time = 0.0
                 min_query_time = 0.0
-                logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors)))
+                logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(query_vectors)))
                 for i in range(run_count):
                     logger.info("Start run query, run %d of %s" % (i+1, run_count))
                     start_time = time.time()
-                    query_res = milvus.query(vectors, top_k, search_param=search_param)
+                    vector_query = {"vector": {vec_field_name: {
+                        "topk": top_k, 
+                        "query": query_vectors, 
+                        "metric_type": utils.metric_type_trans(metric_type), 
+                        "params": search_param}
+                    }}
+                    query_res = milvus.query(vector_query, filter_query=filter_query)
                     interval_time = time.time() - start_time
                     if (i == 0) or (min_query_time > interval_time):
                         min_query_time = interval_time
@@ -212,23 +286,20 @@ class Runner(object):
         end_time = time.time()
         return end_time - start_time
 
-    def do_query_ids(self, milvus, collection_name, top_k, nq, search_param=None):
+    def do_query_ids(self, milvus, collection_name, vec_field_name, top_k, nq, search_param=None, filter_query=None):
         (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
         base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type)
-        vectors = base_query_vectors[0:nq]
-        logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors)))
-        query_res = milvus.query(vectors, top_k, search_param=search_param)
-        result_ids = []
-        result_distances = []
-        for result in query_res:
-            tmp = []
-            tmp_distance = []
-            for item in result:
-                tmp.append(item.id)
-                tmp_distance.append(item.distance)
-            result_ids.append(tmp)
-            result_distances.append(tmp_distance)
-        return result_ids, result_distances
+        query_vectors = base_query_vectors[0:nq]
+        logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(query_vectors)))
+        vector_query = {"vector": {vec_field_name: {
+            "topk": top_k, 
+            "query": query_vectors, 
+            "metric_type": utils.metric_type_trans(metric_type), 
+            "params": search_param}
+        }}
+        query_res = milvus.query(vector_query, filter_query=filter_query)
+        result_ids = milvus.get_ids(query_res)
+        return result_ids
 
     def do_query_acc(self, milvus, collection_name, top_k, nq, id_store_name, search_param=None):
         (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
@@ -286,3 +357,13 @@ class Runner(object):
         d = a[0]
         true_ids = a.reshape(-1, d + 1)[:, 1:].copy()
         return true_ids
+
+    def get_fields(self, milvus, collection_name):
+        fields = []
+        info = milvus.get_info(collection_name)
+        for item in info["fields"]:
+            fields.append(item["name"])
+        return fields
+
+    # def get_filter_query(self, filter_query):
+        # for filter in filter_query:

tests/milvus_benchmark/runners/__init__.py

@@ -0,0 +1,11 @@
+
+
+class BaseRunner(object):
+    def __init__(self):
+        pass
+
+    def set_up(self):
+        pass
+
+    def tear_down(self):
+        pass

tests/milvus_benchmark/runners/locust_runner.py

@@ -0,0 +1,75 @@
+import time
+import random
+from locust import Locust, TaskSet, events, task, between
+from client import MilvusClient
+from . import BasicRunner
+
+
+dim = 128
+top_k = 10
+X = [[random.random() for i in range(dim)] for i in range(1)]
+search_param = {"nprobe": 16}
+
+
+class MilvusTask(object):
+    def __init__(self, type="single", args):
+        self.type = type
+        self.m = None
+        if type == "single":
+            self.m = MilvusClient(host=args["host"], port=args["port"], collection_name=args["collection_name"])
+        elif type == "multi":
+            self.m = MilvusClient(host=args["m"])
+
+    def query(self, *args, **kwargs):
+        name = "milvus_search"
+        request_type = "grpc"
+        start_time = time.time()
+        try:
+            # result = self.m.getattr(*args, **kwargs)
+            status, result = self.m.query(*args, **kwargs)
+        except Exception as e:
+            total_time = int((time.time() - start_time) * 1000)
+            events.request_failure.fire(request_type=request_type, name=name, response_time=total_time, exception=e, response_length=0)
+        else:
+            if not status.OK:
+                total_time = int((time.time() - start_time) * 1000)
+                events.request_failure.fire(request_type=request_type, name=name, response_time=total_time, exception=e, response_length=0)
+            else:
+                total_time = int((time.time() - start_time) * 1000)
+                events.request_success.fire(request_type=request_type, name=name, response_time=total_time, response_length=0)
+                # In this example, I've hardcoded response_length=0. If we would want the response length to be
+                # reported correctly in the statistics, we would probably need to hook in at a lower level
+
+
+class MilvusLocust(Locust):
+    def __init__(self, *args, **kwargs):
+        super(MilvusLocust, self).__init__(*args, **kwargs)
+        self.client = MilvusTask(self.host, self.port, self.collection_name)
+
+
+class Query(MilvusLocust):
+    host = "192.168.1.183"
+    port = 19530
+    collection_name = "sift_128_euclidean"
+    # m = MilvusClient(host=host, port=port, collection_name=collection_name)
+    wait_time = between(0.001, 0.002)
+
+    class task_set(TaskSet):
+        @task
+        def query(self):
+            self.client.query(X, top_k, search_param)
+
+
+class LocustRunner(BasicRunner):
+    """Only one client, not support M/S mode"""
+    def __init__(self, args):
+        # Start client with params including client number && last time && hatch rate ...
+        pass
+
+    def set_up(self):
+        # helm install locust client
+        pass
+
+    def tear_down(self):
+        # helm uninstall
+        pass

tests/milvus_benchmark/scheduler/010_data.json

@@ -0,0 +1,65 @@
+[
+    {
+        "server": "athena",
+        "suite_params": [
+            {
+                "suite": "080_gpu_accuracy.yaml",
+                "image_type": "gpu"
+            },
+            {
+                "suite": "080_search_stability.yaml",
+                "image_type": "gpu"
+            },
+            {
+                "suite": "gpu_accuracy_ann.yaml",
+                "image_type": "gpu"
+            }
+        ]
+    },
+    {
+        "server": "poseidon",
+        "suite_params": [
+            {
+                "suite": "080_gpu_search.yaml",
+                "image_type": "gpu"
+            },
+            {
+                "suite": "080_cpu_search.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "080_gpu_build.yaml",
+                "image_type": "gpu"
+            },
+            {
+                "suite": "080_cpu_accuracy.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "locust_search.yaml",
+                "image_type": "cpu"
+            }
+        ]
+    },
+    {
+        "server": "apollo",
+        "suite_params": [
+            {
+                "suite": "cpu_accuracy_ann.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "080_cpu_build.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "080_insert_performance.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "add_flush_performance.yaml",
+                "image_type": "cpu"
+            }
+        ]
+    }
+]

tests/milvus_benchmark/scheduler/011_data.json

@@ -0,0 +1,53 @@
+[
+    {
+        "server": "apollo",
+        "suite_params": [
+            {
+                "suite": "011_cpu_accuracy_ann.yaml",
+                "image_type": "cpu"
+            }
+        ]
+    },
+    {
+        "server": "eros",
+        "suite_params": [
+            {
+                "suite": "locust_mix.yaml",
+                "image_type": "gpu"
+            },
+            {   
+                "suite": "011_gpu_accuracy.yaml",
+                "image_type": "gpu"
+            }
+        ]
+    },
+    {
+        "server": "poseidon",
+        "suite_params": [
+            {
+                "suite": "011_gpu_search.yaml",
+                "image_type": "gpu"
+            },
+            {
+                "suite": "011_cpu_search.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "011_gpu_build.yaml",
+                "image_type": "gpu"
+            },
+            {
+                "suite": "011_cpu_accuracy.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "011_locust_search.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "011_gpu_accuracy_ann.yaml",
+                "image_type": "gpu"
+            }
+        ]
+    }
+]

tests/milvus_benchmark/scheduler/011_data_acc_debug.json

@@ -0,0 +1,11 @@
+[
+  {
+        "server": "apollo",
+        "suite_params": [
+            {
+                "suite": "011_cpu_accuracy_ann.yaml",
+                "image_type": "cpu"
+            }
+        ]
+    }
+]

tests/milvus_benchmark/scheduler/011_data_gpu_build.json

@@ -1,11 +1,11 @@
 [
-    {
+  {
         "server": "eros",
         "suite_params": [
             {
-                "suite": "070_ann.yaml",
+                "suite": "011_gpu_build_sift10m.yaml",
                 "image_type": "gpu"
             }
         ]
     }
-]
+]

tests/milvus_benchmark/scheduler/011_data_insert.json

@@ -0,0 +1,11 @@
+[
+    {
+        "server": "eros",
+        "suite_params": [
+            {
+                "suite": "011_insert_data.yaml",
+                "image_type": "cpu"
+            }
+        ]
+    }
+  ]

tests/milvus_benchmark/scheduler/011_data_search_debug.json

@@ -0,0 +1,11 @@
+[
+  {
+        "server": "athena",
+        "suite_params": [
+          {
+            "suite": "011_gpu_search_debug.yaml",
+            "image_type": "gpu"
+          }
+        ]
+    }
+]

tests/milvus_benchmark/scheduler/011_delete.json

@@ -0,0 +1,15 @@
+[
+  {
+        "server": "apollo",
+        "suite_params": [
+            {
+                "suite": "011_insert_performance.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "011_delete_performance.yaml",
+                "image_type": "cpu"
+            }
+        ]
+    }
+]

tests/milvus_benchmark/scheduler/080_data.json

@@ -5,6 +5,14 @@
             {
                 "suite": "080_gpu_accuracy.yaml",
                 "image_type": "gpu"
+            },
+            {
+                "suite": "080_search_stability.yaml",
+                "image_type": "gpu"
+            },
+            {
+                "suite": "gpu_accuracy_ann.yaml",
+                "image_type": "gpu"
             }
         ]
     },
@@ -28,7 +36,7 @@
                 "image_type": "cpu"
             },
             {
-                "suite": "080_cpu_build.yaml",
+                "suite": "locust_search.yaml",
                 "image_type": "cpu"
             }
         ]
@@ -41,23 +49,17 @@
                 "image_type": "cpu"
             },
             {
-                "suite": "080_cpu_search_stability.yaml",
+                "suite": "080_cpu_build.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "080_insert_performance.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "add_flush_performance.yaml",
                 "image_type": "cpu"
             }
         ]
-    },
-    {
-        "server": "eros",
-        "suite_params": [
-            {
-                "suite": "gpu_accuracy_ann.yaml",
-                "image_type": "gpu"
-            },
-            {
-                "suite": "080_gpu_stability.yaml",
-                "image_type": "gpu"
-            }
-        ]
     }
-
 ]

tests/milvus_benchmark/scheduler/apollo.json

@@ -3,11 +3,7 @@
         "server": "apollo",
         "suite_params": [
             {
-                "suite": "070_insert_10m.yaml",
-                "image_type": "cpu"
-            },
-            {
-                "suite": "070_cpu_build.yaml",
+                "suite": "011_cpu_accuracy_ann.yaml",
                 "image_type": "cpu"
             }
         ]

tests/milvus_benchmark/scheduler/athena.json

@@ -3,8 +3,16 @@
         "server": "athena",
         "suite_params": [
             {
-                "suite": "add_flush_performance.yaml",
-                "image_type": "cpu"
+                "suite": "011_gpu_accuracy.yaml",
+                "image_type": "gpu"
+            },
+            {
+                "suite": "011_search_stability.yaml",
+                "image_type": "gpu"
+            },
+            {
+                "suite": "011_gpu_accuracy_ann.yaml",
+                "image_type": "gpu"
             }
         ]
     }

tests/milvus_benchmark/scheduler/build.json

@@ -1,9 +1,9 @@
 [
     {
-        "server": "athena",
+        "server": "eros",
         "suite_params": [
             {
-                "suite": "debug_build.yaml",
+                "suite": "011_gpu_build_sift1b.yaml",
                 "image_type": "gpu"
             }
         ]

tests/milvus_benchmark/scheduler/cqx.json

@@ -3,7 +3,7 @@
         "server": "apollo",
         "suite_params": [
             {
-                "suite": "crud_add_flush.yaml",
+                "suite": "011_insert_debug.yaml",
                 "image_type": "cpu"
             }
         ]

tests/milvus_benchmark/scheduler/crud_search.json

@@ -1,11 +0,0 @@
-[
-    {
-        "server": "athena",
-        "suite_params": [
-            {
-                "suite": "crud_search.yaml",
-                "image_type": "gpu"
-            }
-        ]
-    }
-]

tests/milvus_benchmark/scheduler/debug.json

@@ -3,8 +3,8 @@
         "server": "poseidon",
         "suite_params": [
             {
-                "suite": "debug.yaml",
-                "image_type": "gpu"
+                "suite": "clean.yaml",
+                "image_type": "cpu"
             }
         ]
     }

tests/milvus_benchmark/scheduler/eros.json

@@ -1,10 +1,10 @@
 [
     {
-        "server": "eros",
+        "server": "poseidon",
         "suite_params": [
             {
-                "suite": "080_gpu_search_id.yaml",
-                "image_type": "gpu"
+                "suite": "011_locust_search.yaml",
+                "image_type": "cpu"
             }
         ]
     }

tests/milvus_benchmark/scheduler/filter.json

@@ -3,10 +3,9 @@
         "server": "poseidon",
         "suite_params": [
             {
-                "suite": "070_gpu_build.yaml",
+                "suite": "011_search_dsl.yaml",
                 "image_type": "gpu"
             }
         ]
     }
 ]
-

tests/milvus_benchmark/scheduler/insert.json

@@ -1,10 +1,9 @@
 [
     {
-        "server": "poseidon",
         "suite_params": [
             {
-                "suite": "insert_performance.yaml",
-                "image_type": "gpu"
+                "suite": "080_insert_performance.yaml",
+                "image_type": "cpu"
             }
         ]
     }

tests/milvus_benchmark/scheduler/jaccard.json

@@ -1,9 +1,9 @@
 [
     {
-        "server": "poseidon",
+        "server": "athena",
         "suite_params": [
             {
-                "suite": "cpu_search_performance_jaccard.yaml",
+                "suite": "011_cpu_search_binary.yaml",
                 "image_type": "cpu"
             }
         ]

tests/milvus_benchmark/scheduler/locust.json

@@ -3,7 +3,7 @@
         "server": "poseidon",
         "suite_params": [
             {
-                "suite": "crud_add.yaml",
+                "suite": "011_locust_search.yaml",
                 "image_type": "cpu"
             }
         ]

tests/milvus_benchmark/scheduler/locust_mix_debug.json

@@ -1,9 +1,8 @@
 [
     {
-        "server": "poseidon",
         "suite_params": [
             {
-                "suite": "cpu_accuracy_ann_crud_debug.yaml",
+                "suite": "locust_mix.yaml",
                 "image_type": "cpu"
             }
         ]

tests/milvus_benchmark/scheduler/loop.json

@@ -1,9 +1,8 @@
 [
     {
-        "server": "eros",
         "suite_params": [
             {
-                "suite": "070_stability.yaml",
+                "suite": "loop_stability.yaml",
                 "image_type": "gpu"
             }
         ]

tests/milvus_benchmark/scheduler/poseidon.json

@@ -3,23 +3,7 @@
         "server": "poseidon",
         "suite_params": [
             {
-                "suite": "080_gpu_search.yaml",
-                "image_type": "gpu"
-            },
-            {
-                "suite": "080_cpu_search.yaml",
-                "image_type": "cpu"
-            },
-            {
-                "suite": "080_gpu_build.yaml",
-                "image_type": "gpu"
-            },
-            {
-                "suite": "080_cpu_accuracy.yaml",
-                "image_type": "cpu"
-            },
-            {
-                "suite": "080_cpu_build.yaml",
+                "suite": "011_cpu_search_binary.yaml",
                 "image_type": "cpu"
             }
         ]

tests/milvus_benchmark/scheduler/search.json

@@ -1,9 +1,9 @@
 [
     {
-        "server": "eros",
+        "server": "athena",
         "suite_params": [
             {
-                "suite": "debug.yaml",
+                "suite": "011_cpu_search_sift1b.yaml",
                 "image_type": "cpu"
             }
         ]

tests/milvus_benchmark/scheduler/search_performance.json

@@ -1,11 +0,0 @@
-[
-    {
-        "server": "athena",
-        "suite_params": [
-            {
-                "suite": "070_gpu_search.yaml",
-                "image_type": "gpu"
-            }
-        ]
-    }
-]

tests/milvus_benchmark/scheduler/shards.json

@@ -0,0 +1,18 @@
+[
+    {
+        "suite_params": [
+            {
+                "suite": "shards_insert_performance.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "shards_ann_debug.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "shards_loop_stability.yaml",
+                "image_type": "cpu"
+            }
+        ]
+    }
+]

tests/milvus_benchmark/scheduler/shards_ann.json

@@ -0,0 +1,10 @@
+[
+    {
+        "suite_params": [
+            {
+                "suite": "shards_ann_debug.yaml",
+                "image_type": "cpu"
+            }
+        ]
+    }
+]

tests/milvus_benchmark/scheduler/shards_debug.json

@@ -0,0 +1,15 @@
+[
+    {
+        "server": "apollo",
+        "suite_params": [
+            {
+                "suite": "shards_insert_performance_sift1m.yaml",
+                "image_type": "cpu"
+            },
+            {
+                "suite": "shards_search_performance_sift1m.yaml",
+                "image_type": "cpu"
+            }
+        ]
+    }
+]

tests/milvus_benchmark/scheduler/shards_stability.json

@@ -0,0 +1,10 @@
+[
+    {
+        "suite_params": [
+            {
+                "suite": "shards_loop_stability.yaml",
+                "image_type": "cpu"
+            }
+        ]
+    }
+]

tests/milvus_benchmark/scheduler/yx.json

@@ -0,0 +1,11 @@
+[
+    {
+          "server": "athena",
+          "suite_params": [
+              {
+                  "suite": "011_gpu_sift50m_ivf.yaml",
+                  "image_type": "gpu"
+              }
+          ]
+      }
+  ]

tests/milvus_benchmark/scripts/default_config.json

@@ -1,86 +0,0 @@
-[
-    {
-        "job_name": "milvus-apollo",
-        "build_params": {
-            "SUITE": "cpu_accuracy_ann.yaml",
-            "IMAGE_TYPE": "cpu",
-            "IMAGE_VERSION": "master",
-            "SERVER_HOST": "apollo"
-        }
-    },
-    {
-        "job_name": "milvus-apollo",
-        "build_params": {
-            "SUITE": "cpu_stability_sift50m.yaml",
-            "IMAGE_TYPE": "cpu",
-            "IMAGE_VERSION": "master",
-            "SERVER_HOST": "apollo"
-        }
-    },
-
-    {
-        "job_name": "milvus-eros",
-        "build_params": {
-            "SUITE": "gpu_accuracy_ann.yaml",
-            "IMAGE_TYPE": "gpu",
-            "IMAGE_VERSION": "master",
-            "SERVER_HOST": "eros"
-        }
-    },
-    {
-        "job_name": "milvus-eros",
-        "build_params": {
-            "SUITE": "gpu_search_stability.yaml",
-            "IMAGE_TYPE": "gpu",
-            "IMAGE_VERSION": "master",
-            "SERVER_HOST": "eros"
-        }
-    },
-    {
-        "job_name": "milvus-eros",
-        "build_params": {
-            "SUITE": "gpu_build_performance.yaml",
-            "IMAGE_TYPE": "gpu",
-            "IMAGE_VERSION": "master",
-            "SERVER_HOST": "eros"
-        }
-    },
-
-    {
-        "job_name": "milvus-poseidon",
-        "build_params": {
-            "SUITE": "gpu_search_performance.yaml",
-            "IMAGE_TYPE": "gpu",
-            "IMAGE_VERSION": "master",
-            "SERVER_HOST": "poseidon"
-        }
-    },
-    {
-        "job_name": "milvus-poseidon",
-        "build_params": {
-            "SUITE": "cpu_search_performance.yaml",
-            "IMAGE_TYPE": "cpu",
-            "IMAGE_VERSION": "master",
-            "SERVER_HOST": "poseidon"
-        }
-    },
-    {
-        "job_name": "milvus-poseidon",
-        "build_params": {
-            "SUITE": "insert_performance.yaml",
-            "IMAGE_TYPE": "gpu",
-            "IMAGE_VERSION": "master",
-            "SERVER_HOST": "poseidon"
-        }
-    },
-    {
-        "job_name": "milvus-poseidon",
-        "build_params": {
-            "SUITE": "gpu_accuracy.yaml",
-            "IMAGE_TYPE": "gpu",
-            "IMAGE_VERSION": "master",
-            "SERVER_HOST": "poseidon"
-        }
-    }
-
-]

tests/milvus_benchmark/scripts/scheduler.py

@@ -1,28 +0,0 @@
-import json
-from pprint import pprint
-import jenkins
-
-JENKINS_URL = "****"
-server = jenkins.Jenkins(JENKINS_URL, username='****', password='****')
-user = server.get_whoami()
-version = server.get_version()
-print('Hello %s from Jenkins %s' % (user['fullName'], version))
-
-
-# print(job_config)
-# build_params = {
-#     "SUITE": "gpu_accuracy_ann_debug.yaml",
-#     "IMAGE_TYPE": "cpu",
-#     "IMAGE_VERSION": "tanimoto_distance",
-#     "SERVER_HOST": "eros"
-# }
-# print(server.build_job(job_name, build_params))
-
-
-with open("default_config.json") as json_file:
-	data = json.load(json_file)
-	for config in data:
-		build_params = config["build_params"]
-		job_name = config["job_name"]
-		res = server.build_job(job_name, build_params)
-		print(job_name, res)

tests/milvus_benchmark/search_task.py

@@ -0,0 +1,49 @@
+import random
+import logging
+from locust import User, task, between
+from locust_task import MilvusTask
+from client import MilvusClient
+from milvus import DataType
+import utils
+
+connection_type = "single"
+host = "192.168.1.112"
+port = 19530
+collection_name = "sift_10m_100000_128_l2"
+dim = 128
+segment_row_limit = 10000
+m = MilvusClient(host=host, port=port, collection_name=collection_name)
+# m.clean_db()
+# m.create_collection(dim, segment_row_limit, data_type=DataType.FLOAT_VECTOR, auto_id=True, other_fields=None)
+vectors = [[random.random() for _ in range(dim)] for _ in range(1000)]
+entities = m.generate_entities(vectors)
+ids = [i for i in range(10000000)]
+
+class QueryTask(User):
+    wait_time = between(0.001, 0.002)
+    if connection_type == "single":
+        client = MilvusTask(m=m)
+    else:
+        client = MilvusTask(host=host, port=port, collection_name=collection_name)
+
+    # def query(self):
+    #     top_k = 10
+    #     X = [[random.random() for i in range(dim)] for i in range(1)]
+    #     search_param = {"nprobe": 16}
+    #     self.client.query(X, top_k, search_param)
+
+
+    @task(1)
+    def insert(self):
+        self.client.insert(entities)
+
+    # @task(1)
+    # def create(self):
+    #     collection_name = utils.get_unique_name(prefix="locust")
+    #     self.client.create_collection(dim, segment_row_limit, data_type=DataType.FLOAT_VECTOR, auto_id=True, collection_name=collection_name, other_fields=None)
+
+    # @task(1)
+    # def delete(self):
+    #     delete_ids = random.sample(ids, 100)
+    #     logging.error(delete_ids)
+    #     self.client.delete(delete_ids)

tests/milvus_benchmark/suites/011_add_flush_performance.yaml

@@ -0,0 +1,20 @@
+insert_flush_performance:
+  collections:
+    - 
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_2m_128_128_l2_flush
+        cache_config.cpu_cache_capacity: 8
+        cache_config.insert_buffer_size: 2
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false 
+        gpu_resource_config.cache_capacity: 4
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        db_config.auto_flush_interval: 300
+      collection_name: sift_2m_128_128_l2
+      ni_per: 100000

tests/milvus_benchmark/suites/011_build_debug.yaml

@@ -0,0 +1,92 @@
+build_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_hnsw
+        cache_config.cpu_cache_capacity: 4GB
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      collection_name: sift_10m_2000000_128_l2
+      index_type: hnsw
+      index_param:
+        M: 48
+        efConstruction: 500
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_ivf_flat_16384
+    #     cache_config.cpu_cache_capacity: 8GB
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 100
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 6GB
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   collection_name: sift_50m_4000000_128_l2
+    #   index_type: ivf_flat
+    #   index_param:
+    #     nlist: 16384
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8_16384
+    #     cache_config.cpu_cache_capacity: 8GB
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 100
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 6GB
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   collection_name: sift_50m_4000000_128_l2
+    #   index_type: ivf_sq8
+    #   index_param:
+    #     nlist: 16384
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8h_16384
+    #     cache_config.cpu_cache_capacity: 8GB
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 100
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 6GB
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   collection_name: sift_50m_4000000_128_l2
+    #   index_type: ivf_sq8h
+    #   index_param:
+    #     nlist: 16384
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_pq_16384
+    #     cache_config.cpu_cache_capacity: 8GB
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 100
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 6GB
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   collection_name: sift_50m_4000000_128_l2
+    #   index_type: ivf_pq
+    #   index_param:
+    #     nlist: 16384
+    #     m: 32

tests/milvus_benchmark/suites/011_compact_performance.yaml

@@ -0,0 +1,15 @@
+compact_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_1m_128_128_l2_delete
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      collection_name: sift_1m_50000_128_l2_2

tests/milvus_benchmark/suites/011_cpu_accuracy.yaml

@@ -0,0 +1,55 @@
+accuracy:
+  collections:
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8
+        cache_config.cpu_cache_capacity: 8GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      collection_name: sift_10m_100000_128_l2
+      top_ks: [64]
+      nqs: [1000]
+      search_params:
+        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_sq8
+        cache_config.cpu_cache_capacity: 8GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      collection_name: sift_10m_2000000_128_l2
+      top_ks: [64]
+      nqs: [1000]
+      search_params:
+        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_hnsw
+        cache_config.cpu_cache_capacity: 8GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      collection_name: sift_10m_2000000_128_l2
+      top_ks: [64]
+      nqs: [1000]
+      search_params:
+        ef: [64, 100, 200, 500, 700]

tests/milvus_benchmark/suites/011_cpu_accuracy_ann.yaml

@@ -0,0 +1,271 @@
+ann_accuracy:
+  collections:
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      collection_name: sift_128_euclidean
+      segment_row_limits: [500000]
+      index_types: ['flat']
+      index_params:
+        nlist: [16384]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 512, 16384]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      collection_name: sift_128_euclidean
+      segment_row_limits: [500000]
+      index_types: ['ivf_flat', 'ivf_sq8']
+      index_params:
+        nlist: [16384]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      collection_name: sift_128_euclidean
+      segment_row_limits: [500000]
+      index_types: ['ivf_pq']
+      index_params:
+        nlist: [16384]
+        m: [32]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      collection_name: sift_128_euclidean
+      segment_row_limits: [500000]
+      index_types: ['annoy']
+      index_params:
+        n_trees: [8, 32]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        search_k: [50, 100, 500, 1000]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      collection_name: sift_128_euclidean
+      segment_row_limits: [500000]
+      index_types: ['hnsw']
+      index_params:
+        M: [16]
+        efConstruction: [500]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        ef: [16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5
+      collection_name: glove_200_angular
+      segment_row_limits: [500000]
+      index_types: ['flat']
+      index_params:
+        nlist: [16384]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 512, 16384]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5
+      collection_name: glove_200_angular
+      segment_row_limits: [500000]
+      index_types: ['ivf_flat', 'ivf_sq8']
+      index_params:
+        nlist: [16384]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5
+      collection_name: glove_200_angular
+      segment_row_limits: [500000]
+      index_types: ['ivf_pq']
+      index_params:
+        nlist: [16384]
+        m: [20]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+
+    - milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5
+      collection_name: glove_200_angular
+      segment_row_limits: [500000]
+      index_types: ['annoy']
+      index_params:
+        n_trees: [8, 32]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        search_k: [50, 100, 500, 1000]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5
+      collection_name: glove_200_angular
+      segment_row_limits: [500000]
+      index_types: ['hnsw']
+      index_params:
+        M: [36]
+        efConstruction: [500]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        ef: [10, 16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5
+      collection_name: glove_200_angular
+      segment_row_limits: [500000]
+      index_types: ['nsg']
+      index_params:
+        search_length: 45
+        out_degree: 50
+        candidate_pool_size: 300
+        knng: 100
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        search_length: [50]

tests/milvus_benchmark/suites/011_cpu_accuracy_ann_debug.yaml

@@ -0,0 +1,73 @@
+ann_accuracy:
+  collections:
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      server:
+        cpus: 50.0
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      collection_name: sift_128_euclidean
+      segment_row_limits: [500000]
+      index_types: ['flat']
+      index_params:
+        nlist: [16384]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 512]
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      collection_name: sift_128_euclidean
+      segment_row_limits: [500000]
+      index_types: ['flat']
+      index_params:
+        nlist: [16384]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 512]
+#    -
+#      server:
+#        cache_config.cpu_cache_capacity: 16GB
+#        engine_config.use_blas_threshold: 1100
+#        engine_config.gpu_search_threshold: 1
+#        gpu_resource_config.enable: false
+#        gpu_resource_config.cache_capacity: 4GB
+#        gpu_resource_config.search_resources:
+#          - gpu0
+#          - gpu1
+#        gpu_resource_config.build_index_resources:
+#          - gpu0
+#          - gpu1
+#      source_file: /test/milvus/ann_hdf5/kosarak-27983-jaccard.hdf5
+#      collection_name: kosarak_27984_jaccard
+#      segment_row_limits: [100000]
+#      index_types: ['bin_flat', 'bin_ivf_flat']
+#      index_params:
+#        nlist: [2048]
+#      top_ks: [10]
+#      nqs: [10000]
+#      search_params:
+#        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]

tests/milvus_benchmark/suites/011_cpu_accuracy_rhnsw.yaml

@@ -0,0 +1,36 @@
+accuracy:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_rhnsw_pq
+        cache_config.cpu_cache_capacity: 8GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      collection_name: sift_10m_2000000_128_l2
+      top_ks: [32]
+      nqs: [1000]
+      search_params:
+        ef: [32, 64, 100]
+#    -
+#      server:
+#        db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_rhnsw_sq
+#        cache_config.cpu_cache_capacity: 8GB
+#        engine_config.use_blas_threshold: 1100
+#        engine_config.gpu_search_threshold: 1
+#        gpu_resource_config.enable: false
+#        gpu_resource_config.cache_capacity: 4GB
+#        gpu_resource_config.search_resources:
+#          - gpu0
+#        gpu_resource_config.build_index_resources:
+#          - gpu0
+#      collection_name: sift_50m_4000000_128_l2
+#      top_ks: [64]
+#      nqs: [1000]
+#      search_params:
+#        ef: [32, 64, 100, 200, 500]

tests/milvus_benchmark/suites/011_cpu_build.yaml

@@ -0,0 +1,40 @@
+build_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/random_1m_1024_512_l2_ivf
+        cache_config.cpu_cache_capacity: 32
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      collection_name: random_1m_1024_512_l2
+      index_type: ivf_flat
+      index_param:
+        nlist: 16384 
+
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_1m_128_128_l2_pq
+        cache_config.cpu_cache_capacity: 32
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      collection_name: sift_1m_128_128_l2
+      index_type: ivf_pq
+      index_param:
+        nlist: 8092
+        m: 32

tests/milvus_benchmark/suites/011_cpu_build_binary.yaml

@@ -0,0 +1,11 @@
+build_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/binary_50m_4000000_512_jaccard_ivf
+        cache_config.cpu_cache_capacity: 8GB
+        gpu_resource_config.enable: false
+      collection_name: binary_50m_4000000_512_jaccard
+      index_type: bin_ivf_flat
+      index_param:
+        nlist: 2048

tests/milvus_benchmark/suites/011_cpu_build_hnsw.yaml

@@ -0,0 +1,12 @@
+build_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_ip_hnsw
+        cache_config.cpu_cache_capacity: 8GB
+        gpu_resource_config.enable: false
+      collection_name: sift_10m_2000000_128_ip
+      index_type: hnsw
+      index_param:
+        M: 48
+        efConstruction: 500

tests/milvus_benchmark/suites/011_cpu_build_rhnsw.yaml

@@ -0,0 +1,23 @@
+build_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_rhnsw_pq
+        cache_config.cpu_cache_capacity: 8GB
+        gpu_resource_config.enable: false
+      collection_name: sift_10m_2000000_128_l2
+      index_type: rhnsw_pq
+      index_param:
+        M: 48
+        efConstruction: 500
+        PQM: 16
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_rhnsw_sq
+        cache_config.cpu_cache_capacity: 8GB
+        gpu_resource_config.enable: false
+      collection_name: sift_50m_4000000_128_l2
+      index_type: rhnsw_sq
+      index_param:
+        M: 48
+        efConstruction: 500

tests/milvus_benchmark/suites/011_cpu_search.yaml

@@ -0,0 +1,255 @@
+search_performance:
+  collections:
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_sq8
+        cache_config.cpu_cache_capacity: 150GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_1b_4000000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 200, 500, 1000]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2
+        cache_config.cpu_cache_capacity: 64GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_10m_2000000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 200, 500, 1000]
+      search_params:
+        -
+          nprobe: 8
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2
+        cache_config.cpu_cache_capacity: 64GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_50m_4000000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 200, 500, 1000]
+      search_params:
+        -
+          nprobe: 8
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_ivf_flat
+        cache_config.cpu_cache_capacity: 64GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_50m_4000000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 200, 500, 1000]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_50m_4000000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 200, 500, 1000]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_pq
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+        wal_enable: true
+      collection_name: sift_10m_2000000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 200, 500, 1000]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_ivf_flat
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+        wal_enable: true
+      collection_name: sift_10m_2000000_128_l2
+      run_count: 2
+      top_ks: [1, 1000]
+      nqs: [1, 100, 1000]
+      search_params:
+        -
+          nprobe: 8
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_nsg
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+        wal_enable: true
+      collection_name: sift_10m_100000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 200, 500, 1000]
+      search_params:
+        -
+         search_length: 50
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_annoy
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 200
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+        wal_enable: true
+      collection_name: sift_10m_100000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100]
+      nqs: [1, 10, 100, 200, 500, 1000]
+      search_params:
+        -
+         search_k: 100
+        -
+         search_k: 500
+        -
+         search_k: 1000
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/binary_50m_4000000_512_jaccard
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: binary_50m_4000000_512_jaccard
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/binary_50m_4000000_512_jaccard_ivf
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: binary_50m_4000000_512_jaccard
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32

tests/milvus_benchmark/suites/011_cpu_search_binary.yaml

@@ -0,0 +1,49 @@
+search_performance:
+  collections:
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/binary_50m_4000000_512_jaccard
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: binary_50m_4000000_512_jaccard
+      run_count: 2
+      top_ks: [10, 1, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/binary_50m_4000000_512_jaccard_ivf
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: binary_50m_4000000_512_jaccard
+      run_count: 2
+      top_ks: [10, 1, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32

tests/milvus_benchmark/suites/011_cpu_search_sift10m.yaml

@@ -0,0 +1,122 @@
+search_performance:
+  collections:
+#    -
+#      milvus:
+#        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2
+#        cache_config.cpu_cache_capacity: 32GB
+#        engine_config.use_blas_threshold: 0
+#        engine_config.gpu_search_threshold: 100
+#        gpu_resource_config.enable: true
+#        gpu_resource_config.cache_capacity: 6GB
+#        gpu_resource_config.search_resources:
+#          - gpu0
+#          - gpu1
+#        gpu_resource_config.build_index_resources:
+#          - gpu0
+#          - gpu1
+#        wal_enable: true
+#      collection_name: sift_10m_2000000_128_l2
+#      run_count: 2
+#      top_ks: [1, 1000]
+#      nqs: [1, 100, 1200]
+#      search_params:
+#        -
+#          nprobe: 8
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      server:
+        cpus: 24.0
+      collection_name: sift_10m_2000000_128_l2
+      run_count: 2
+      top_ks: [1, 1000]
+      nqs: [1, 100, 1200]
+      search_params:
+        -
+          nprobe: 8
+#    -
+#      server:
+#        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_ivf_flat_16384
+#        cache_config.cpu_cache_capacity: 32GB
+#        engine_config.use_blas_threshold: 0
+#        engine_config.gpu_search_threshold: 100
+#        gpu_resource_config.enable: true
+#        gpu_resource_config.cache_capacity: 6GB
+#        gpu_resource_config.search_resources:
+#          - gpu0
+#          - gpu1
+#        gpu_resource_config.build_index_resources:
+#          - gpu0
+#          - gpu1
+#        wal_enable: true
+#      collection_name: sift_10m_2000000_128_l2
+#      run_count: 2
+#      top_ks: [1, 10, 100, 1000]
+#      nqs: [1, 10, 100, 1000, 1200]
+#      search_params:
+#        -
+#          nprobe: 8
+#        -
+#          nprobe: 32
+#
+#    -
+#      server:
+#        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_sq8_16384
+#        cache_config.cpu_cache_capacity: 32GB
+#        engine_config.use_blas_threshold: 0
+#        engine_config.gpu_search_threshold: 100
+#        gpu_resource_config.enable: true
+#        gpu_resource_config.cache_capacity: 6GB
+#        gpu_resource_config.search_resources:
+#          - gpu0
+#          - gpu1
+#        gpu_resource_config.build_index_resources:
+#          - gpu0
+#          - gpu1
+#        wal_enable: true
+#      collection_name: sift_10m_2000000_128_l2
+#      run_count: 2
+#      top_ks: [1, 10, 100, 1000]
+#      nqs: [1, 10, 100, 1000, 1200]
+#      search_params:
+#        -
+#          nprobe: 8
+#        -
+#          nprobe: 32
+#
+#    -
+#      server:
+#        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_pq_16384
+#        cache_config.cpu_cache_capacity: 32GB
+#        engine_config.use_blas_threshold: 0
+#        engine_config.gpu_search_threshold: 100
+#        gpu_resource_config.enable: true
+#        gpu_resource_config.cache_capacity: 6GB
+#        gpu_resource_config.search_resources:
+#          - gpu0
+#          - gpu1
+#        gpu_resource_config.build_index_resources:
+#          - gpu0
+#          - gpu1
+#        wal_enable: true
+#      collection_name: sift_10m_2000000_128_l2
+#      run_count: 2
+#      top_ks: [1, 10, 100, 1000]
+#      nqs: [1, 10, 100, 1000, 1200]
+#      search_params:
+#        -
+#          nprobe: 8
+#        -
+#          nprobe: 32

tests/milvus_benchmark/suites/011_cpu_search_sift10m_100k.yaml

@@ -0,0 +1,97 @@
+search_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_10m_100000_128_l2
+      run_count: 2
+      top_ks: [1, 1000]
+      nqs: [1, 100, 1200]
+      search_params:
+        -
+          nprobe: 8
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_ivf_flat
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_10m_100000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32
+
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_10m_100000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_pq
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_10m_100000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32

tests/milvus_benchmark/suites/011_cpu_search_sift10m_filter.yaml

@@ -0,0 +1,97 @@
+search_performance:
+  collections:
+     -
+       server:
+         db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2/
+         cache_config.cpu_cache_capacity: 32GB
+         engine_config.use_blas_threshold: 0
+         engine_config.gpu_search_threshold: 100
+         gpu_resource_config.enable: true
+         gpu_resource_config.cache_capacity: 6GB
+         gpu_resource_config.search_resources:
+           - gpu0
+           - gpu1
+         gpu_resource_config.build_index_resources:
+           - gpu0
+           - gpu1
+         wal_enable: true
+       collection_name: sift_10m_2000000_128_l2
+       run_count: 2
+       top_ks: [1, 1000]
+       nqs: [1, 100, 1200]
+       search_params:
+         -
+           nprobe: 8
+     -
+       server:
+         db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_ivf_flat
+         cache_config.cpu_cache_capacity: 32GB
+         engine_config.use_blas_threshold: 0
+         engine_config.gpu_search_threshold: 100
+         gpu_resource_config.enable: true
+         gpu_resource_config.cache_capacity: 6GB
+         gpu_resource_config.search_resources:
+           - gpu0
+           - gpu1
+         gpu_resource_config.build_index_resources:
+           - gpu0
+           - gpu1
+         wal_enable: true
+       collection_name: sift_10m_2000000_128_l2
+       run_count: 2
+       top_ks: [1, 10, 100, 1000]
+       nqs: [1, 10, 100, 1000, 1200]
+       search_params:
+         -
+           nprobe: 8
+         -
+           nprobe: 32
+
+     -
+       server:
+         db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_sq8
+         cache_config.cpu_cache_capacity: 32GB
+         engine_config.use_blas_threshold: 0
+         engine_config.gpu_search_threshold: 100
+         gpu_resource_config.enable: true
+         gpu_resource_config.cache_capacity: 6GB
+         gpu_resource_config.search_resources:
+           - gpu0
+           - gpu1
+         gpu_resource_config.build_index_resources:
+           - gpu0
+           - gpu1
+         wal_enable: true
+       collection_name: sift_10m_2000000_128_l2
+       run_count: 2
+       top_ks: [1, 10, 100, 1000]
+       nqs: [1, 10, 100, 1000, 1200]
+       search_params:
+         -
+           nprobe: 8
+         -
+           nprobe: 32
+     -
+       server:
+         db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_pq
+         cache_config.cpu_cache_capacity: 32GB
+         engine_config.use_blas_threshold: 0
+         engine_config.gpu_search_threshold: 100
+         gpu_resource_config.enable: true
+         gpu_resource_config.cache_capacity: 6GB
+         gpu_resource_config.search_resources:
+           - gpu0
+           - gpu1
+         gpu_resource_config.build_index_resources:
+           - gpu0
+           - gpu1
+         wal_enable: true
+       collection_name: sift_10m_2000000_128_l2
+       run_count: 2
+       top_ks: [1, 10, 100, 1000]
+       nqs: [1, 10, 100, 1000, 1200]
+       search_params:
+         -
+           nprobe: 8
+         -
+           nprobe: 32

tests/milvus_benchmark/suites/011_cpu_search_sift10m_hnsw.yaml

@@ -0,0 +1,40 @@
+search_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_hnsw
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        wal_enable: true
+      collection_name: sift_10m_2000000_128_l2
+      run_count: 2
+      top_ks: [100]
+      nqs: [1, 10, 100, 200, 500, 1000]
+      search_params:
+        -
+          ef: 100
+        -
+          ef: 200
+        -
+          ef: 500
+        -
+          ef: 1000
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_ip_hnsw
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        wal_enable: true
+      collection_name: sift_10m_2000000_128_ip
+      run_count: 2
+      top_ks: [100]
+      nqs: [1, 10, 100, 200, 500, 1000]
+      search_params:
+        -
+          ef: 100
+        -
+          ef: 200
+        -
+          ef: 500
+        -
+          ef: 1000

tests/milvus_benchmark/suites/011_cpu_search_sift10m_ivf.yaml

@@ -0,0 +1,32 @@
+search_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_ivf_flat_16384
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        wal_enable: true
+      collection_name: sift_10m_2000000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_ip_ivf_flat_16384
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        wal_enable: true
+      collection_name: sift_10m_2000000_128_ip
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32

tests/milvus_benchmark/suites/011_cpu_search_sift1b.yaml

@@ -0,0 +1,26 @@
+search_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_sq8
+        cache_config.cpu_cache_capacity: 150GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_1b_4000000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32

tests/milvus_benchmark/suites/011_cpu_search_sift50m.yaml

@@ -0,0 +1,98 @@
+search_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_50m_4000000_128_l2
+      run_count: 2
+      top_ks: [1, 1000]
+      nqs: [1, 100, 1200]
+      search_params:
+        -
+          nprobe: 8
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_ivf_flat_16384
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_50m_4000000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32
+
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_sq8_16384
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_50m_4000000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32
+
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_50m_4000000_128_l2_pq_16384
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+        wal_enable: true
+      collection_name: sift_50m_4000000_128_l2
+      run_count: 2
+      top_ks: [1, 10, 100, 1000]
+      nqs: [1, 10, 100, 1000, 1200]
+      search_params:
+        -
+          nprobe: 8
+        -
+          nprobe: 32

tests/milvus_benchmark/suites/011_delete_performance.yaml

@@ -0,0 +1,17 @@
+delete_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_1m_128_128_l2_delete
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+      collection_name: sift_1m_50000_128_l2_2
+      ni_per: 50000
+      auto_flush: false

tests/milvus_benchmark/suites/011_get_id.yaml

@@ -0,0 +1,17 @@
+get_ids_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_1024_128_l2_sq8
+        cache_config.cpu_cache_capacity: 32
+        engine_config.use_blas_threshold: 0
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 6
+        gpu_resource_config.search_resources:
+          - gpu0
+        gpu_resource_config.build_index_resources:
+          - gpu0
+        wal_enable: true
+      collection_name: sift_10m_1024_128_l2
+      ids_length_per_segment: [1, 10, 100] 

tests/milvus_benchmark/suites/011_gpu_accuracy.yaml

@@ -0,0 +1,61 @@
+accuracy:
+  collections:
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2_sq8
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4GB 
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      collection_name: sift_10m_2000000_128_l2
+      top_ks: [64]
+      nqs: [1000]
+      search_params:
+        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_l2
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4GB 
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      collection_name: sift_10m_2000000_128_l2
+      top_ks: [64]
+      nqs: [1000]
+      search_params:
+        nprobe: [1]
+
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_2000000_128_ip
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4GB 
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      collection_name: sift_10m_2000000_128_ip
+      top_ks: [64]
+      nqs: [1000]
+      search_params:
+        nprobe: [1]

tests/milvus_benchmark/suites/011_gpu_accuracy_ann.yaml

@@ -0,0 +1,172 @@
+ann_accuracy:
+  collections:
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      collection_name: sift_128_euclidean
+      segment_row_limits: [100000]
+      index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h']
+      index_params:
+        nlist: [16384]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      collection_name: sift_128_euclidean
+      segment_row_limits: [100000]
+      index_types: ['ivf_pq']
+      index_params:
+        nlist: [16384]
+        m: [32]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      collection_name: sift_128_euclidean
+      segment_row_limits: [100000]
+      index_types: ['hnsw']
+      index_params:
+        M: [16]
+        efConstruction: [500]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        ef: [16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true 
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5
+      collection_name: glove_200_angular
+      segment_row_limits: [100000]
+      index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h']
+      index_params:
+        nlist: [16384]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5
+      collection_name: glove_200_angular
+      segment_row_limits: [100000]
+      index_types: ['hnsw']
+      index_params:
+        M: [36]
+        efConstruction: [500]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        ef: [10, 16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/kosarak-27983-jaccard.hdf5
+      collection_name: kosarak_27984_jaccard
+      segment_row_limits: [100000]
+      index_types: ['bin_flat', 'bin_ivf_flat']
+      index_params:
+        nlist: [2048]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
+
+    -
+      milvus:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: false
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-256-hamming.hdf5
+      collection_name: sift_256_hamming
+      segment_row_limits: [100000]
+      index_types: ['bin_flat', 'bin_ivf_flat']
+      index_params:
+        nlist: [2048]
+      top_ks: [100]
+      nqs: [1000]
+      search_params:
+        nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]

tests/milvus_benchmark/suites/011_gpu_accuracy_ann_debug.yaml

@@ -0,0 +1,25 @@
+ann_accuracy:
+  collections:
+    -
+      server:
+        cache_config.cpu_cache_capacity: 16GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 4GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5
+      collection_name: sift_128_euclidean
+      segment_row_limits: [100000]
+      index_types: ['ivf_sq8']
+      index_params:
+        nlist: [16384]
+      top_ks: [10]
+      nqs: [10000]
+      search_params:
+        nprobe: [1, 512]

tests/milvus_benchmark/suites/011_gpu_accuracy_debug.yaml

@@ -0,0 +1,23 @@
+accuracy:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_1b_4000000_128_l2_sq8
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+          - gpu2
+          - gpu3
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      collection_name: sift_1b_4000000_128_l2
+      top_ks: [64]
+      nqs: [1000]
+      search_params:
+        nprobe: [32, 64, 128]

tests/milvus_benchmark/suites/011_gpu_build.yaml

@@ -0,0 +1,21 @@
+build_performance:
+  collections:
+    -
+      milvus:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8_4096
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+      collection_name: sift_10m_100000_128_l2
+      index_type: ivf_sq8
+      index_param:
+        nlist: 4096
+

tests/milvus_benchmark/suites/011_gpu_build_debug.yaml

@@ -0,0 +1,151 @@
+build_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8_4096
+        cache_config.cpu_cache_capacity: 32GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 1
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu2
+      collection_name: sift_10m_100000_128_l2
+      index_type: ivf_sq8
+      index_param:
+        nlist: 4096
+
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_ivf
+    #     cache_config.cpu_cache_capacity: 8GB
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4GB
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   collection_name: sift_10m_100000_128_l2
+    #   index_type: ivf_flat
+    #   index_param:
+    #     nlist: 1024
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8h
+    #     cache_config.cpu_cache_capacity: 8GB
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4GB
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   collection_name: sift_10m_100000_128_l2
+    #   index_type: ivf_sq8h
+    #   index_param:
+    #     nlist: 1024
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8
+    #     cache_config.cpu_cache_capacity: 8GB
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4GB
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   collection_name: sift_10m_100000_128_l2
+    #   index_type: ivf_sq8
+    #   index_param:
+    #     nlist: 1024
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_pq
+    #     cache_config.cpu_cache_capacity: 8GB
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4GB
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   collection_name: sift_10m_100000_128_l2
+    #   index_type: ivf_pq
+    #   index_param:
+    #     nlist: 1024
+    #     m: 32
+    # -
+    #   server:
+    #     db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_hnsw
+    #     cache_config.cpu_cache_capacity: 8GB
+    #     engine_config.use_blas_threshold: 1100
+    #     engine_config.gpu_search_threshold: 1
+    #     gpu_resource_config.enable: true
+    #     gpu_resource_config.cache_capacity: 4GB
+    #     gpu_resource_config.search_resources:
+    #       - gpu0
+    #       - gpu1
+    #     gpu_resource_config.build_index_resources:
+    #       - gpu0
+    #       - gpu1
+    #   collection_name: sift_10m_100000_128_l2
+    #   index_type: hnsw
+    #   index_param:
+    #     M: 48
+    #     efConstruction: 500
+#    -
+#      server:
+#        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_annoy
+#        cache_config.cpu_cache_capacity: 8GB
+#        engine_config.use_blas_threshold: 1100
+#        engine_config.gpu_search_threshold: 1
+#        gpu_resource_config.enable: true
+#        gpu_resource_config.cache_capacity: 4GB
+#        gpu_resource_config.search_resources:
+#          - gpu0
+#          - gpu1
+#        gpu_resource_config.build_index_resources:
+#          - gpu0
+#          - gpu1
+#      collection_name: sift_10m_100000_128_l2
+#      index_type: annoy
+#      index_param:
+#        n_trees: 8
+#    -
+#      server:
+#        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_nsg
+#        cache_config.cpu_cache_capacity: 8GB
+#        engine_config.use_blas_threshold: 1100
+#        engine_config.gpu_search_threshold: 1
+#        gpu_resource_config.enable: true
+#        gpu_resource_config.cache_capacity: 4GB
+#        gpu_resource_config.search_resources:
+#          - gpu0
+#          - gpu1
+#        gpu_resource_config.build_index_resources:
+#          - gpu0
+#          - gpu1
+#      collection_name: sift_10m_100000_128_l2
+#      index_type: nsg
+#      index_param:
+#        search_length: 50
+#        out_degree: 40
+#        candidate_pool_size: 100
+#        knng: 50

tests/milvus_benchmark/suites/011_gpu_build_sift10m.yaml

@@ -0,0 +1,148 @@
+build_performance:
+  collections:
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_ivf_flat
+        cache_config.cpu_cache_capacity: 8GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+          - gpu2
+          - gpu3
+      collection_name: sift_10m_2000000_128_l2
+      index_type: ivf_flat
+      index_param:
+        nlist: 1024
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_sq8
+        cache_config.cpu_cache_capacity: 8GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+          - gpu2
+          - gpu3
+      collection_name: sift_10m_2000000_128_l2
+      index_type: ivf_sq8
+      index_param:
+        nlist: 1024
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_sq8h
+        cache_config.cpu_cache_capacity: 8GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+          - gpu2
+          - gpu3
+      collection_name: sift_10m_2000000_128_l2
+      index_type: ivf_sq8h
+      index_param:
+        nlist: 1024
+    -
+      server:
+        db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_2000000_128_l2_pq
+        cache_config.cpu_cache_capacity: 8GB
+        engine_config.use_blas_threshold: 1100
+        engine_config.gpu_search_threshold: 100
+        gpu_resource_config.enable: true
+        gpu_resource_config.cache_capacity: 6GB
+        gpu_resource_config.search_resources:
+          - gpu0
+          - gpu1
+        gpu_resource_config.build_index_resources:
+          - gpu0
+          - gpu1
+          - gpu2
+          - gpu3
+      collection_name: sift_10m_2000000_128_l2
+      index_type: ivf_pq
+      index_param:
+        nlist: 1024
+        m: 32
+#    -
+#      server:
+#        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_hnsw
+#        cache_config.cpu_cache_capacity: 8GB
+#        engine_config.use_blas_threshold: 1100
+#        engine_config.gpu_search_threshold: 1
+#        gpu_resource_config.enable: true
+#        gpu_resource_config.cache_capacity: 4GB
+#        gpu_resource_config.search_resources:
+#          - gpu0
+#          - gpu1
+#        gpu_resource_config.build_index_resources:
+#          - gpu0
+#          - gpu1
+#          - gpu2
+#          - gpu3
+#      collection_name: sift_10m_100000_128_l2
+#      index_type: hnsw
+#      index_param:
+#        M: 48
+#        efConstruction: 500
+#    -
+#      server:
+#        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_annoy
+#        cache_config.cpu_cache_capacity: 8GB
+#        engine_config.use_blas_threshold: 1100
+#        engine_config.gpu_search_threshold: 100
+#        gpu_resource_config.enable: true
+#        gpu_resource_config.cache_capacity: 4GB
+#        gpu_resource_config.search_resources:
+#          - gpu0
+#          - gpu1
+#        gpu_resource_config.build_index_resources:
+#          - gpu0
+#          - gpu1
+#          - gpu2
+#          - gpu3
+#      collection_name: sift_10m_100000_128_l2
+#      index_type: annoy
+#      index_param:
+#        n_trees: 8
+#    -
+#      server:
+#        db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_nsg
+#        cache_config.cpu_cache_capacity: 8GB
+#        engine_config.use_blas_threshold: 1100
+#        engine_config.gpu_search_threshold: 100
+#        gpu_resource_config.enable: true
+#        gpu_resource_config.cache_capacity: 4GB
+#        gpu_resource_config.search_resources:
+#          - gpu0
+#          - gpu1
+#        gpu_resource_config.build_index_resources:
+#          - gpu0
+#          - gpu1
+#          - gpu2
+#          - gpu3
+#      collection_name: sift_10m_100000_128_l2
+#      index_type: nsg
+#      index_param:
+#        search_length: 50
+#        out_degree: 40
+#        candidate_pool_size: 100
+#        knng: 50
+