Merge branch '0.5.0' into 0.5.1

Former-commit-id: a04484fa5b0adfc77ea742c4d7c38a52e8209603

CHANGELOG.md

@@ -27,11 +27,13 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-654 - Describe index timeout when building index
 - MS-658 - Fix SQ8 Hybrid can't search
 - MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources
+- \#9 - Change default gpu_cache_capacity to 4
 - \#20 - C++ sdk example get grpc error 
 - \#23 - Add unittest to improve code coverage
 - \#31 - make clang-format failed after run build.sh -l
 - \#39 - Create SQ8H index hang if using github server version
 - \#30 - Some troubleshoot messages in Milvus do not provide enough information
+- \#48 - Config unittest failed
 
 ## Improvement
 - MS-552 - Add and change the easylogging library
@@ -53,6 +55,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-626 - Refactor DataObj to support cache any type data
 - MS-648 - Improve unittest
 - MS-655 - Upgrade SPTAG
+- \#42 - Put union of index_build_device and search resources to gpu_pool
 
 ## New Feature
 - MS-614 - Preload table at startup
@@ -74,6 +77,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-624 - Re-organize project directory for open-source
 - MS-635 - Add compile option to support customized faiss
 - MS-660 - add ubuntu_build_deps.sh
+- \#18 - Add all test cases
 	
 # Milvus 0.4.0 (2019-09-12)
 

ci/jenkins/Jenkinsfile

@@ -0,0 +1,152 @@
+pipeline {
+    agent none
+
+    options {
+        timestamps()
+    }
+
+    parameters{
+        choice choices: ['Release', 'Debug'], description: '', name: 'BUILD_TYPE'
+        string defaultValue: 'cf1434e7-5a4b-4d25-82e8-88d667aef9e5', description: 'GIT CREDENTIALS ID', name: 'GIT_CREDENTIALS_ID', trim: true
+        string defaultValue: 'registry.zilliz.com', description: 'DOCKER REGISTRY URL', name: 'DOKCER_REGISTRY_URL', trim: true
+        string defaultValue: 'ba070c98-c8cc-4f7c-b657-897715f359fc', description: 'DOCKER CREDENTIALS ID', name: 'DOCKER_CREDENTIALS_ID', trim: true
+        string defaultValue: 'http://192.168.1.202/artifactory/milvus', description: 'JFROG ARTFACTORY URL', name: 'JFROG_ARTFACTORY_URL', trim: true
+        string defaultValue: '1a527823-d2b7-44fd-834b-9844350baf14', description: 'JFROG CREDENTIALS ID', name: 'JFROG_CREDENTIALS_ID', trim: true
+    }
+
+    environment {
+        PROJECT_NAME = "milvus"
+        LOWER_BUILD_TYPE = params.BUILD_TYPE.toLowerCase()
+        SEMVER = "${BRANCH_NAME}"
+        JOBNAMES = env.JOB_NAME.split('/')
+        PIPELINE_NAME = "${JOBNAMES[0]}"
+    }
+
+    stages {
+        stage("Ubuntu 18.04") {
+            environment {
+                OS_NAME = "ubuntu18.04"
+                PACKAGE_VERSION = VersionNumber([
+                    versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-ubuntu18.04-x86_64-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
+                ]);
+                DOCKER_VERSION = "${SEMVER}-${OS_NAME}-${LOWER_BUILD_TYPE}"
+            }
+
+            stages {
+                stage("Run Build") {
+                    agent {
+                        kubernetes {
+                            label 'build'
+                            defaultContainer 'jnlp'
+                            yamlFile 'ci/jenkins/pod/milvus-build-env-pod.yaml'
+                        }
+                    }
+
+                    stages {
+                        stage('Build') {
+                            steps {
+                                container('milvus-build-env') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/build.groovy"
+                                    }
+                                }
+                            }
+                        }
+                        stage('Code Coverage') {
+                            steps {
+                                container('milvus-build-env') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/coverage.groovy"
+                                    }
+                                }
+                            }
+                        }
+                        stage('Upload Package') {
+                            steps {
+                                container('milvus-build-env') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/package.groovy"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                stage("Publish docker images") {
+                    agent {
+                        kubernetes {
+                            label 'publish'
+                            defaultContainer 'jnlp'
+                            yamlFile 'ci/jenkins/pod/docker-pod.yaml'
+                        }
+                    }
+
+                    stages {
+                        stage('Publish') {
+                            steps {
+                                container('publish-images'){
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/publishImages.groovy"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                stage("Deploy to Development") {
+                    agent {
+                        kubernetes {
+                            label 'dev-test'
+                            defaultContainer 'jnlp'
+                            yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
+                        }
+                    }
+
+                    stages {
+                        stage("Deploy to Dev") {
+                            steps {
+                                container('milvus-test-env') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy"
+                                    }
+                                }
+                            }
+                        }
+
+                        stage("Dev Test") {
+                            steps {
+                                container('milvus-test-env') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/singleDevTest.groovy"
+                                    }
+                                }
+                            }
+                        }
+
+                        stage ("Cleanup Dev") {
+                            steps {
+                                container('milvus-test-env') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    post {
+                        unsuccessful {
+                            container('milvus-test-env') {
+                                script {
+                                    load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+

ci/jenkins/jenkinsfile/build.groovy

@@ -0,0 +1,9 @@
+timeout(time: 60, unit: 'MINUTES') {
+    dir ("ci/jenkins/scripts") {
+        sh "./build.sh -l"
+        withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
+            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -d /opt/milvus -j -u -c"
+        }
+    }
+}
+

ci/jenkins/jenkinsfile/cleanupSingleDev.groovy

@@ -0,0 +1,9 @@
+try {
+    sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu"
+} catch (exc) {
+    def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu", returnStatus: true
+    if (!helmResult) {
+        sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu"
+    }
+    throw exc
+}

ci/jenkins/jenkinsfile/coverage.groovy

@@ -0,0 +1,10 @@
+timeout(time: 60, unit: 'MINUTES') {
+    dir ("ci/jenkins/scripts") {
+        sh "./coverage.sh -o /opt/milvus -u root -p 123456 -t \$POD_IP"
+        // Set some env variables so codecov detection script works correctly
+        withCredentials([[$class: 'StringBinding', credentialsId: "${env.PIPELINE_NAME}-codecov-token", variable: 'CODECOV_TOKEN']]) {
+            sh 'curl -s https://codecov.io/bash | bash -s - -f output_new.info || echo "Codecov did not collect coverage reports"'
+        }
+    }
+}
+

ci/jenkins/jenkinsfile/deploySingle2Dev.groovy

@@ -0,0 +1,14 @@
+try {
+    sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+    sh 'helm repo update'
+    dir ('milvus-helm') {
+        checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
+        dir ("milvus-gpu") {
+            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/values.yaml --namespace milvus ."
+        }
+    }
+} catch (exc) {
+    echo 'Helm running failed!'
+    sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu"
+    throw exc
+}

ci/jenkins/jenkinsfile/package.groovy

@@ -0,0 +1,9 @@
+timeout(time: 5, unit: 'MINUTES') {
+    sh "tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz -C /opt/ milvus"
+    withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
+        def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz")
+        if (uploadStatus != 0) {
+            error("\" ${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" upload to \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" failed!")
+        }
+    }
+}

ci/jenkins/jenkinsfile/publishImages.groovy

@@ -0,0 +1,47 @@
+container('publish-images') {
+    timeout(time: 15, unit: 'MINUTES') {
+        dir ("docker/deploy/${OS_NAME}") {
+            def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz"
+
+            withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
+                def downloadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage}")
+
+                if (downloadStatus != 0) {
+                    error("\" Download \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage} \" failed!")
+                }
+            }
+            sh "tar zxvf ${binaryPackage}"
+            def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}"
+
+            try {
+                def isExistSourceImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName} 2>&1 > /dev/null")
+                if (isExistSourceImage == 0) {
+                    def removeSourceImageStatus = sh(returnStatus: true, script: "docker rmi ${imageName}")
+                }
+
+                def customImage = docker.build("${imageName}")
+
+                def isExistTargeImage = sh(returnStatus: true, script: "docker inspect --type=image ${params.DOKCER_REGISTRY_URL}/${imageName} 2>&1 > /dev/null")
+                if (isExistTargeImage == 0) {
+                    def removeTargeImageStatus = sh(returnStatus: true, script: "docker rmi ${params.DOKCER_REGISTRY_URL}/${imageName}")
+                }
+
+                docker.withRegistry("https://${params.DOKCER_REGISTRY_URL}", "${params.DOCKER_CREDENTIALS_ID}") {
+                    customImage.push()
+                }
+            } catch (exc) {
+                throw exc
+            } finally {
+                def isExistSourceImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName} 2>&1 > /dev/null")
+                if (isExistSourceImage == 0) {
+                    def removeSourceImageStatus = sh(returnStatus: true, script: "docker rmi ${imageName}")
+                }
+
+                def isExistTargeImage = sh(returnStatus: true, script: "docker inspect --type=image ${params.DOKCER_REGISTRY_URL}/${imageName} 2>&1 > /dev/null")
+                if (isExistTargeImage == 0) {
+                    def removeTargeImageStatus = sh(returnStatus: true, script: "docker rmi ${params.DOKCER_REGISTRY_URL}/${imageName}")
+                }
+            }
+        } 
+    }
+}

ci/jenkins/jenkinsfile/singleDevTest.groovy

@@ -0,0 +1,22 @@
+timeout(time: 30, unit: 'MINUTES') {
+    dir ("tests/milvus_python_test") {
+        sh 'python3 -m pip install -r requirements.txt'
+        sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local"
+    }
+    // mysql database backend test
+    load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
+
+    if (!fileExists('milvus-helm')) {
+        dir ("milvus-helm") {
+            checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
+        }
+    }
+    dir ("milvus-helm") {
+        dir ("milvus-gpu") {
+            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml --namespace milvus ."
+        }
+    }
+    dir ("tests/milvus_python_test") {
+        sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local"
+    }
+}

ci/jenkins/pod/docker-pod.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: publish
+    componet: docker
+spec:
+  containers:
+  - name: publish-images
+    image: registry.zilliz.com/library/docker:v1.0.0
+    securityContext:
+      privileged: true
+    command:
+    - cat
+    tty: true
+    volumeMounts:
+    - name: docker-sock
+      mountPath: /var/run/docker.sock
+  volumes:
+  - name: docker-sock
+    hostPath:
+      path: /var/run/docker.sock
+

ci/jenkins/pod/milvus-build-env-pod.yaml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: milvus-build-env
+  labels:
+    app: milvus
+    componet: build-env
+spec:
+  containers:
+  - name: milvus-build-env
+    image: registry.zilliz.com/milvus/milvus-build-env:v0.5.0-ubuntu18.04
+    env:
+    - name: POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    command:
+    - cat
+    tty: true
+    resources:
+      limits:
+        memory: "32Gi"
+        cpu: "8.0"
+        nvidia.com/gpu: 1
+      requests:
+        memory: "16Gi"
+        cpu: "4.0"
+  - name: milvus-mysql
+    image: mysql:5.6
+    env:
+    - name: MYSQL_ROOT_PASSWORD
+      value: 123456
+    ports:
+    - containerPort: 3306
+      name: mysql

ci/jenkins/pod/testEnvironment.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus
+    componet: test-env
+spec:
+  containers:
+  - name: milvus-test-env
+    image: registry.zilliz.com/milvus/milvus-test-env:v0.1
+    command:
+    - cat
+    tty: true
+    volumeMounts:
+    - name: kubeconf
+      mountPath: /root/.kube/
+      readOnly: true
+  volumes:
+  - name: kubeconf
+    secret:
+      secretName: test-cluster-config
+

ci/jenkins/scripts/build.sh

@@ -0,0 +1,142 @@
+#!/bin/bash
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build"
+BUILD_TYPE="Debug"
+BUILD_UNITTEST="OFF"
+INSTALL_PREFIX="/opt/milvus"
+BUILD_COVERAGE="OFF"
+DB_PATH="/opt/milvus"
+PROFILING="OFF"
+USE_JFROG_CACHE="OFF"
+RUN_CPPLINT="OFF"
+CUSTOMIZATION="OFF" # default use ori faiss
+CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+
+CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
+wget -q --method HEAD ${CUSTOMIZED_FAISS_URL}
+if [ $? -eq 0 ]; then
+  CUSTOMIZATION="ON"
+else
+  CUSTOMIZATION="OFF"
+fi
+
+while getopts "o:d:t:ulcgjhx" arg
+do
+        case $arg in
+             o)
+                INSTALL_PREFIX=$OPTARG
+                ;;
+             d)
+                DB_PATH=$OPTARG
+                ;;
+             t)
+                BUILD_TYPE=$OPTARG # BUILD_TYPE
+                ;;
+             u)
+                echo "Build and run unittest cases" ;
+                BUILD_UNITTEST="ON";
+                ;;
+             l)
+                RUN_CPPLINT="ON"
+                ;;
+             c)
+                BUILD_COVERAGE="ON"
+                ;;
+             g)
+                PROFILING="ON"
+                ;;
+             j)
+                USE_JFROG_CACHE="ON"
+                ;;
+             x)
+                CUSTOMIZATION="OFF" # force use ori faiss
+                ;;
+             h) # help
+                echo "
+
+parameter:
+-o: install prefix(default: /opt/milvus)
+-d: db data path(default: /opt/milvus)
+-t: build type(default: Debug)
+-u: building unit test options(default: OFF)
+-l: run cpplint, clang-format and clang-tidy(default: OFF)
+-c: code coverage(default: OFF)
+-g: profiling(default: OFF)
+-j: use jfrog cache build directory(default: OFF)
+-h: help
+
+usage:
+./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-h]
+                "
+                exit 0
+                ;;
+             ?)
+                echo "ERROR! unknown argument"
+        exit 1
+        ;;
+        esac
+done
+
+if [[ ! -d ${CMAKE_BUILD_DIR} ]]; then
+    mkdir ${CMAKE_BUILD_DIR}
+fi
+
+cd ${CMAKE_BUILD_DIR}
+
+# remove make cache since build.sh -l use default variables
+# force update the variables each time
+make rebuild_cache
+
+CMAKE_CMD="cmake \
+-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
+-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
+-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
+-DBUILD_COVERAGE=${BUILD_COVERAGE} \
+-DMILVUS_DB_PATH=${DB_PATH} \
+-DMILVUS_ENABLE_PROFILING=${PROFILING} \
+-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
+-DCUSTOMIZATION=${CUSTOMIZATION} \
+-DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
+.."
+echo ${CMAKE_CMD}
+${CMAKE_CMD}
+
+if [[ ${RUN_CPPLINT} == "ON" ]]; then
+    # cpplint check
+    make lint
+    if [ $? -ne 0 ]; then
+        echo "ERROR! cpplint check failed"
+        exit 1
+    fi
+    echo "cpplint check passed!"
+
+    # clang-format check
+    make check-clang-format
+    if [ $? -ne 0 ]; then
+        echo "ERROR! clang-format check failed"
+        exit 1
+    fi
+    echo "clang-format check passed!"
+
+#    # clang-tidy check
+#    make check-clang-tidy
+#    if [ $? -ne 0 ]; then
+#        echo "ERROR! clang-tidy check failed"
+#        rm -f CMakeCache.txt
+#        exit 1
+#    fi
+#    echo "clang-tidy check passed!"
+else
+    # compile and build
+    make -j8 || exit 1
+    make install || exit 1
+fi

ci/jenkins/scripts/coverage.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+INSTALL_PREFIX="/opt/milvus"
+CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build"
+MYSQL_USER_NAME=root
+MYSQL_PASSWORD=123456
+MYSQL_HOST='127.0.0.1'
+MYSQL_PORT='3306'
+
+while getopts "o:u:p:t:h" arg
+do
+        case $arg in
+             o)
+                INSTALL_PREFIX=$OPTARG
+                ;;
+             u)
+                MYSQL_USER_NAME=$OPTARG
+                ;;
+             p)
+                MYSQL_PASSWORD=$OPTARG
+                ;;
+             t)
+                MYSQL_HOST=$OPTARG
+                ;;
+             h) # help
+                echo "
+
+parameter:
+-o: milvus install prefix(default: /opt/milvus)
+-u: mysql account
+-p: mysql password
+-t: mysql host
+-h: help
+
+usage:
+./coverage.sh -o \${INSTALL_PREFIX} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h]
+                "
+                exit 0
+                ;;
+             ?)
+                echo "ERROR! unknown argument"
+        exit 1
+        ;;
+        esac
+done
+
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${INSTALL_PREFIX}/lib
+
+LCOV_CMD="lcov"
+# LCOV_GEN_CMD="genhtml"
+
+FILE_INFO_BASE="base.info"
+FILE_INFO_MILVUS="server.info"
+FILE_INFO_OUTPUT="output.info"
+FILE_INFO_OUTPUT_NEW="output_new.info"
+DIR_LCOV_OUTPUT="lcov_out"
+
+DIR_GCNO="${CMAKE_BUILD_DIR}"
+DIR_UNITTEST="${INSTALL_PREFIX}/unittest"
+
+# delete old code coverage info files
+rm -rf lcov_out
+rm -f FILE_INFO_BASE FILE_INFO_MILVUS FILE_INFO_OUTPUT FILE_INFO_OUTPUT_NEW
+
+MYSQL_DB_NAME=milvus_`date +%s%N`
+
+function mysql_exc()
+{
+    cmd=$1
+    mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}"
+    if [ $? -ne 0 ]; then
+        echo "mysql $cmd run failed"
+    fi
+}
+
+mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};"
+mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';"
+mysql_exc "FLUSH PRIVILEGES;"
+mysql_exc "USE ${MYSQL_DB_NAME};"
+
+# get baseline
+${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}"
+if [ $? -ne 0 ]; then
+    echo "gen baseline coverage run failed"
+    exit -1
+fi
+
+for test in `ls ${DIR_UNITTEST}`; do
+    echo $test
+    case ${test} in
+        test_db)
+            # set run args for test_db
+            args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}"
+            ;;
+        *test_*)
+            args=""
+            ;;
+    esac
+    # run unittest
+    ${DIR_UNITTEST}/${test} "${args}"
+    if [ $? -ne 0 ]; then
+        echo ${args}
+        echo ${DIR_UNITTEST}/${test} "run failed"
+    fi
+done
+
+mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};"
+
+# gen code coverage
+${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c
+# merge coverage
+${LCOV_CMD} -a ${FILE_INFO_BASE} -a ${FILE_INFO_MILVUS} -o "${FILE_INFO_OUTPUT}"
+
+# remove third party from tracefiles
+${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
+    "/usr/*" \
+    "*/boost/*" \
+    "*/cmake_build/*_ep-prefix/*" \
+    "*/src/index/cmake_build*" \
+    "*/src/index/thirdparty*" \
+    "*/src/grpc*" \
+    "*/src/metrics/MetricBase.h" \
+    "*/src/server/Server.cpp" \
+    "*/src/server/DBWrapper.cpp" \
+    "*/src/server/grpc_impl/GrpcServer.cpp" \
+    "*/src/utils/easylogging++.h" \
+    "*/src/utils/easylogging++.cc"
+
+# gen html report
+# ${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/

ci/jenkinsfile/milvus_build.groovy

@@ -17,7 +17,7 @@ container('milvus-build-env') {
                             && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \
                             && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u -c"
 
-                            sh "./coverage.sh -u root -p 123456 -t 192.168.1.194"
+                            sh "./coverage.sh -u root -p 123456 -t \$POD_IP"
                         }
                     }
                 } catch (exc) {

ci/main_jenkinsfile

@@ -45,6 +45,11 @@ spec:
   containers:
   - name: milvus-build-env
     image: registry.zilliz.com/milvus/milvus-build-env:v0.13
+    env:
+    - name: POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
     command:
     - cat
     tty: true
@@ -56,6 +61,14 @@ spec:
       requests:
         memory: "14Gi"
         cpu: "5.0"
+  - name: milvus-mysql
+    image: mysql:5.6
+    env:
+    - name: MYSQL_ROOT_PASSWORD
+      value: 123456
+    ports:
+    - containerPort: 3306
+      name: mysql
 """
                         }
                     }

ci/main_jenkinsfile_no_ut

@@ -45,6 +45,11 @@ spec:
   containers:
   - name: milvus-build-env
     image: registry.zilliz.com/milvus/milvus-build-env:v0.13
+    env:
+    - name: POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
     command:
     - cat
     tty: true
@@ -56,6 +61,14 @@ spec:
       requests:
         memory: "14Gi"
         cpu: "5.0"
+  - name: milvus-mysql
+    image: mysql:5.6
+    env:
+    - name: MYSQL_ROOT_PASSWORD
+      value: 123456
+    ports:
+    - containerPort: 3306
+      name: mysql
 """
                         }
                     }

core/coverage.sh

@@ -114,15 +114,15 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
     "/usr/*" \
     "*/boost/*" \
     "*/cmake_build/*_ep-prefix/*" \
-    "src/index/cmake_build*" \
-    "src/index/thirdparty*" \
-    "src/grpc*"\
-    "src/metrics/MetricBase.h"\
-    "src/server/Server.cpp"\
-    "src/server/DBWrapper.cpp"\
-    "src/server/grpc_impl/GrpcServer.cpp"\
-    "src/utils/easylogging++.h"\
-    "src/utils/easylogging++.cc"\
+    "*/src/index/cmake_build*" \
+    "*/src/index/thirdparty*" \
+    "*/src/grpc*" \
+    "*/src/metrics/MetricBase.h" \
+    "*/src/server/Server.cpp" \
+    "*/src/server/DBWrapper.cpp" \
+    "*/src/server/grpc_impl/GrpcServer.cpp" \
+    "*/src/utils/easylogging++.h" \
+    "*/src/utils/easylogging++.cc"
 
 # gen html report
 ${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -121,6 +121,11 @@ ExecutionEngineImpl::HybridLoad() const {
         return;
     }
 
+    if (index_->GetType() == IndexType::FAISS_IDMAP) {
+        ENGINE_LOG_WARNING << "HybridLoad with type FAISS_IDMAP, ignore";
+        return;
+    }
+
     const std::string key = location_ + ".quantizer";
     std::vector<uint64_t> gpus = scheduler::get_gpu_pool();
 
@@ -161,6 +166,9 @@ ExecutionEngineImpl::HybridLoad() const {
         quantizer_conf->mode = 1;
         quantizer_conf->gpu_id = best_device_id;
         auto quantizer = index_->LoadQuantizer(quantizer_conf);
+        if (quantizer == nullptr) {
+            ENGINE_LOG_ERROR << "quantizer is nullptr";
+        }
         index_->SetQuantizer(quantizer);
         auto cache_quantizer = std::make_shared<CachedQuantizer>(quantizer);
         cache::GpuCacheMgr::GetInstance(best_device_id)->InsertItem(key, cache_quantizer);
@@ -172,6 +180,9 @@ ExecutionEngineImpl::HybridUnset() const {
     if (index_type_ != EngineType::FAISS_IVFSQ8H) {
         return;
     }
+    if (index_->GetType() == IndexType::FAISS_IDMAP) {
+        return;
+    }
     index_->UnsetQuantizer();
 }
 

core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -244,6 +244,7 @@ if(CUSTOMIZATION)
         # set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0
         # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
         # set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1
+        # set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1
         set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
     endif()
 else()

core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp

@@ -20,17 +20,22 @@
 #include <faiss/IndexIVF.h>
 #include <faiss/IndexIVFFlat.h>
 #include <faiss/gpu/GpuCloner.h>
+
+#include <chrono>
 #include <memory>
 #include <utility>
 #include <vector>
 
 #include "knowhere/adapter/VectorAdapter.h"
 #include "knowhere/common/Exception.h"
+#include "knowhere/common/Log.h"
 #include "knowhere/index/vector_index/IndexGPUIVF.h"
 #include "knowhere/index/vector_index/IndexIVF.h"
 
 namespace knowhere {
 
+using stdclock = std::chrono::high_resolution_clock;
+
 IndexModelPtr
 IVF::Train(const DatasetPtr& dataset, const Config& config) {
     auto build_cfg = std::dynamic_pointer_cast<IVFCfg>(config);
@@ -212,7 +217,15 @@ IVF::GenGraph(const int64_t& k, Graph& graph, const DatasetPtr& dataset, const C
 void
 IVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) {
     auto params = GenParams(cfg);
+    stdclock::time_point before = stdclock::now();
     faiss::ivflib::search_with_parameters(index_.get(), n, (float*)data, k, distances, labels, params.get());
+    stdclock::time_point after = stdclock::now();
+    double search_cost = (std::chrono::duration<double, std::micro>(after - before)).count();
+    KNOWHERE_LOG_DEBUG << "IVF search cost: " << search_cost
+                       << ", quantization cost: " << faiss::indexIVF_stats.quantization_time
+                       << ", data search cost: " << faiss::indexIVF_stats.search_time;
+    faiss::indexIVF_stats.quantization_time = 0;
+    faiss::indexIVF_stats.search_time = 0;
 }
 
 VectorIndexPtr

core/src/scheduler/SchedInst.cpp

@@ -53,26 +53,32 @@ load_simple_config() {
     config.GetResourceConfigSearchResources(pool);
 
     // get resources
-    bool use_cpu_to_compute = false;
-    for (auto& resource : pool) {
-        if (resource == "cpu") {
-            use_cpu_to_compute = true;
-            break;
-        }
-    }
     auto gpu_ids = get_gpu_pool();
 
+    int32_t build_gpu_id;
+    config.GetResourceConfigIndexBuildDevice(build_gpu_id);
+
     // create and connect
     ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false));
 
     auto io = Connection("io", 500);
-    ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, use_cpu_to_compute));
+    ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true));
     ResMgrInst::GetInstance()->Connect("disk", "cpu", io);
 
     auto pcie = Connection("pcie", 12000);
+    bool find_build_gpu_id = false;
     for (auto& gpu_id : gpu_ids) {
         ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true));
         ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie);
+        if (build_gpu_id == gpu_id) {
+            find_build_gpu_id = true;
+        }
+    }
+
+    if (not find_build_gpu_id) {
+        ResMgrInst::GetInstance()->Add(
+            ResourceFactory::Create(std::to_string(build_gpu_id), "GPU", build_gpu_id, true, true));
+        ResMgrInst::GetInstance()->Connect("cpu", std::to_string(build_gpu_id), pcie);
     }
 }
 

core/src/scheduler/action/PushTaskToNeighbour.cpp

@@ -187,8 +187,8 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr
             Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu);
 
             bool find_gpu_res = false;
-            for (uint64_t i = 0; i < compute_resources.size(); ++i) {
-                if (res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) {
+            if (res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) {
+                for (uint64_t i = 0; i < compute_resources.size(); ++i) {
                     if (compute_resources[i]->name() ==
                         res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu)->name()) {
                         find_gpu_res = true;

core/src/wrapper/VecIndex.h

@@ -25,6 +25,7 @@
 #include "knowhere/common/BinarySet.h"
 #include "knowhere/common/Config.h"
 #include "knowhere/index/vector_index/Quantizer.h"
+#include "utils/Log.h"
 #include "utils/Status.h"
 
 namespace milvus {
@@ -102,6 +103,7 @@ class VecIndex : public cache::DataObj {
     ////////////////
     virtual knowhere::QuantizerPtr
     LoadQuantizer(const Config& conf) {
+        ENGINE_LOG_ERROR << "LoadQuantizer virtual funciton called.";
         return nullptr;
     }
 

core/unittest/server/CMakeLists.txt

@@ -67,11 +67,3 @@ target_link_libraries(test_server
         )
 
 install(TARGETS test_server DESTINATION unittest)
-
-configure_file(appendix/server_config.yaml
-        "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/server_config.yaml"
-        COPYONLY)
-
-configure_file(appendix/log_config.conf
-        "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/log_config.conf"
-        COPYONLY)

core/unittest/server/appendix/log_config.conf

@@ -1,27 +0,0 @@
-* GLOBAL:
-    FORMAT                  =   "%datetime | %level | %logger | %msg"
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log"
-    ENABLED                 =   true
-    TO_FILE                 =   true
-    TO_STANDARD_OUTPUT      =   false
-    SUBSECOND_PRECISION     =   3
-    PERFORMANCE_TRACKING    =   false
-    MAX_LOG_FILE_SIZE       =   209715200 ## Throw log files away after 200MB
-* DEBUG:
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log"
-    ENABLED                 =   true
-* WARNING:
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log"
-* TRACE:
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log"
-* VERBOSE:
-    FORMAT                  =   "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
-    TO_FILE                 =   false
-    TO_STANDARD_OUTPUT      =   false
-## Error logs
-* ERROR:
-    ENABLED                 =   true
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log"
-* FATAL:
-    ENABLED                 =   true
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log"

core/unittest/server/appendix/server_config.yaml

@@ -1,43 +0,0 @@
-# Default values are used when you make no changes to the following parameters.
-
-server_config:
-  address: 0.0.0.0                  # milvus server ip address (IPv4)
-  port: 19530                       # port range: 1025 ~ 65534
-  deploy_mode: single               # deployment type: single, cluster_readonly, cluster_writable
-  time_zone: UTC+8
-
-db_config:
-  primary_path: /tmp/milvus    # path used to store data and meta
-  secondary_path:                   # path used to store data only, split by semicolon
-
-  backend_url: sqlite://:@:/        # URI format: dialect://username:password@host:port/database
-                                    # Keep 'dialect://:@:/', and replace other texts with real values
-                                    # Replace 'dialect' with 'mysql' or 'sqlite'
-
-  insert_buffer_size: 4             # GB, maximum insert buffer size allowed
-                                    # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
-
-  preload_table:                    # preload data at startup, '*' means load all tables, empty value means no preload
-                                    # you can specify preload tables like this: table1,table2,table3
-
-metric_config:
-  enable_monitor: false             # enable monitoring or not
-  collector: prometheus             # prometheus
-  prometheus_config:
-    port: 8080                      # port prometheus uses to fetch metrics
-
-cache_config:
-  cpu_cache_capacity: 16            # GB, CPU memory used for cache
-  cpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered
-  gpu_cache_capacity: 4             # GB, GPU memory used for cache
-  gpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered
-  cache_insert_data: false          # whether to load inserted data into cache
-
-engine_config:
-  use_blas_threshold: 20            # if nq <  use_blas_threshold, use SSE, faster with fluctuated response times
-                                    # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
-
-resource_config:
-  search_resources:                 # define the GPUs used for search computation, valid value: gpux
-    - gpu0
-  index_build_device: gpu0          # GPU used for building index

core/unittest/server/test_config.cpp

@@ -22,28 +22,27 @@
 #include "utils/CommonUtil.h"
 #include "utils/ValidationUtil.h"
 #include "server/Config.h"
+#include "server/utils.h"
 
 namespace {
 
-static const char *CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml";
-static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf";
-
 static constexpr uint64_t KB = 1024;
 static constexpr uint64_t MB = KB * 1024;
 static constexpr uint64_t GB = MB * 1024;
 
 } // namespace
 
-TEST(ConfigTest, CONFIG_TEST) {
+TEST_F(ConfigTest, CONFIG_TEST) {
     milvus::server::ConfigMgr *config_mgr = milvus::server::YamlConfigMgr::GetInstance();
 
     milvus::Status s = config_mgr->LoadConfigFile("");
     ASSERT_FALSE(s.ok());
 
-    s = config_mgr->LoadConfigFile(LOG_FILE_PATH);
+    std::string config_path(CONFIG_PATH);
+    s = config_mgr->LoadConfigFile(config_path+ INVALID_CONFIG_FILE);
     ASSERT_FALSE(s.ok());
 
-    s = config_mgr->LoadConfigFile(CONFIG_FILE_PATH);
+    s = config_mgr->LoadConfigFile(config_path + VALID_CONFIG_FILE);
     ASSERT_TRUE(s.ok());
 
     config_mgr->Print();
@@ -99,9 +98,10 @@ TEST(ConfigTest, CONFIG_TEST) {
     ASSERT_TRUE(seqs.empty());
 }
 
-TEST(ConfigTest, SERVER_CONFIG_TEST) {
+TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
+    std::string config_path(CONFIG_PATH);
     milvus::server::Config &config = milvus::server::Config::GetInstance();
-    milvus::Status s = config.LoadConfigFile(CONFIG_FILE_PATH);
+    milvus::Status s = config.LoadConfigFile(config_path + VALID_CONFIG_FILE);
     ASSERT_TRUE(s.ok());
 
     s = config.ValidateConfig();

core/unittest/server/test_util.cpp

@@ -275,6 +275,11 @@ TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) {
     ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::INVALID).code(),
               milvus::SERVER_INVALID_INDEX_TYPE);
     for (int i = 1; i <= (int)milvus::engine::EngineType::MAX_VALUE; i++) {
+#ifndef CUSTOMIZATION
+        if (i == (int)milvus::engine::EngineType::FAISS_IVFSQ8H) {
+            continue;
+        }
+#endif
         ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS);
     }
     ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(

core/unittest/server/utils.cpp

@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "server/utils.h"
+#include "utils/CommonUtil.h"
+
+#include <fstream>
+#include <iostream>
+#include <thread>
+#include <string>
+
+namespace {
+
+static const char
+    * VALID_CONFIG_STR = "# Default values are used when you make no changes to the following parameters.\n"
+                         "\n"
+                         "server_config:\n"
+                         "  address: 0.0.0.0                  # milvus server ip address (IPv4)\n"
+                         "  port: 19530                       # port range: 1025 ~ 65534\n"
+                         "  deploy_mode: single               \n"
+                         "  time_zone: UTC+8\n"
+                         "\n"
+                         "db_config:\n"
+                         "  primary_path: /tmp/milvus    # path used to store data and meta\n"
+                         "  secondary_path:                   # path used to store data only, split by semicolon\n"
+                         "\n"
+                         "  backend_url: sqlite://:@:/        \n"
+                         "\n"
+                         "  insert_buffer_size: 4             # GB, maximum insert buffer size allowed\n"
+                         "  preload_table:                    \n"
+                         "\n"
+                         "metric_config:\n"
+                         "  enable_monitor: false             # enable monitoring or not\n"
+                         "  collector: prometheus             # prometheus\n"
+                         "  prometheus_config:\n"
+                         "    port: 8080                      # port prometheus uses to fetch metrics\n"
+                         "\n"
+                         "cache_config:\n"
+                         "  cpu_cache_capacity: 16            # GB, CPU memory used for cache\n"
+                         "  cpu_cache_threshold: 0.85         \n"
+                         "  gpu_cache_capacity: 4             # GB, GPU memory used for cache\n"
+                         "  gpu_cache_threshold: 0.85         \n"
+                         "  cache_insert_data: false          # whether to load inserted data into cache\n"
+                         "\n"
+                         "engine_config:\n"
+                         "  use_blas_threshold: 20            \n"
+                         "\n"
+                         "resource_config:\n"
+                         "  search_resources:                 \n"
+                         "    - gpu0\n"
+                         "  index_build_device: gpu0          # GPU used for building index";
+
+static const char* INVALID_CONFIG_STR = "*INVALID*";
+
+void
+WriteToFile(const std::string& file_path, const char* content) {
+    std::fstream fs(file_path.c_str(), std::ios_base::out);
+
+    //write data to file
+    fs << content;
+    fs.close();
+}
+
+} // namespace
+
+
+void
+ConfigTest::SetUp() {
+    std::string config_path(CONFIG_PATH);
+    milvus::server::CommonUtil::CreateDirectory(config_path);
+    WriteToFile(config_path + VALID_CONFIG_FILE, VALID_CONFIG_STR);
+    WriteToFile(config_path+ INVALID_CONFIG_FILE, INVALID_CONFIG_STR);
+}
+
+void
+ConfigTest::TearDown() {
+    std::string config_path(CONFIG_PATH);
+    milvus::server::CommonUtil::DeleteDirectory(config_path);
+}

core/unittest/server/utils.h

@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include <gtest/gtest.h>
+#include <chrono>
+
+static const char *CONFIG_PATH = "/tmp/milvus_test/";
+static const char *VALID_CONFIG_FILE = "valid_config.yaml";
+static const char *INVALID_CONFIG_FILE = "invalid_config.conf";
+
+class ConfigTest : public ::testing::Test {
+ protected:
+    void SetUp() override;
+    void TearDown() override;
+};

docker/build_env/ubuntu16.04/Dockerfile

@@ -0,0 +1,25 @@
+FROM nvidia/cuda:10.1-devel-ubuntu16.04
+
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+
+RUN apt-get update && apt-get install -y --no-install-recommends wget && \
+    wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \
+    wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
+    apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
+    sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \
+    apt-get update && apt-get install -y --no-install-recommends \
+    git flex bison gfortran \
+    curl libtool automake libboost1.58-all-dev libssl-dev pkg-config libcurl4-openssl-dev \
+    clang-format-6.0 clang-tidy-6.0 \
+    lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.4-243 intel-mkl-core-2019.4-243 && \
+    apt-get remove --purge -y && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
+
+RUN sh -c 'echo export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64:\$LD_LIBRARY_PATH > /etc/profile.d/mkl.sh'
+
+COPY docker-entrypoint.sh /app/docker-entrypoint.sh
+ENTRYPOINT [ "/app/docker-entrypoint.sh" ]
+CMD [ "start" ]
+

docker/build_env/ubuntu16.04/docker-entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+if [ "$1" = 'start' ]; then
+   tail -f /dev/null
+fi
+
+exec "$@"
+

docker/build_env/ubuntu18.04/Dockerfile

@@ -0,0 +1,25 @@
+FROM nvidia/cuda:10.1-devel-ubuntu18.04
+
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+
+RUN apt-get update && apt-get install -y --no-install-recommends wget && \
+    wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \
+    wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
+    apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
+    sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \
+    apt-get update && apt-get install -y --no-install-recommends \
+    git flex bison gfortran \
+    curl libtool automake libboost-all-dev libssl-dev pkg-config libcurl4-openssl-dev \
+    clang-format-6.0 clang-tidy-6.0 \
+    lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.4-243 intel-mkl-core-2019.4-243 && \
+    apt-get remove --purge -y && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
+
+RUN sh -c 'echo export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64:\$LD_LIBRARY_PATH > /etc/profile.d/mkl.sh'
+
+COPY docker-entrypoint.sh /app/docker-entrypoint.sh
+ENTRYPOINT [ "/app/docker-entrypoint.sh" ]
+CMD [ "start" ]
+

docker/build_env/ubuntu18.04/docker-entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+if [ "$1" = 'start' ]; then
+   tail -f /dev/null
+fi
+
+exec "$@"
+

docker/deploy/ubuntu16.04/Dockerfile

@@ -0,0 +1,23 @@
+FROM nvidia/cuda:10.1-devel-ubuntu16.04
+
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+
+RUN rm -rf /etc/apt/sources.list.d/nvidia-ml.list && rm -rf /etc/apt/sources.list.d/cuda.list
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gfortran libsqlite3-dev libmysqlclient-dev libcurl4-openssl-dev python3 && \
+    apt-get remove --purge -y && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
+
+COPY ./docker-entrypoint.sh /opt
+COPY ./milvus /opt/milvus
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/milvus/lib"
+
+ENTRYPOINT [ "/opt/docker-entrypoint.sh" ]
+
+CMD [ "start" ]
+
+EXPOSE 19530
+

docker/deploy/ubuntu16.04/docker-entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+if [ "$1" == 'start' ]; then
+    cd /opt/milvus/scripts && ./start_server.sh
+fi
+
+exec "$@"
+

docker/deploy/ubuntu18.04/Dockerfile

@@ -0,0 +1,23 @@
+FROM nvidia/cuda:10.1-devel-ubuntu18.04
+
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+
+RUN rm -rf /etc/apt/sources.list.d/nvidia-ml.list && rm -rf /etc/apt/sources.list.d/cuda.list
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gfortran libsqlite3-dev libmysqlclient-dev libcurl4-openssl-dev python3 && \
+    apt-get remove --purge -y && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
+
+COPY ./docker-entrypoint.sh /opt
+COPY ./milvus /opt/milvus
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/milvus/lib"
+
+ENTRYPOINT [ "/opt/docker-entrypoint.sh" ]
+
+CMD [ "start" ]
+
+EXPOSE 19530
+

docker/deploy/ubuntu18.04/docker-entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+if [ "$1" == 'start' ]; then
+    cd /opt/milvus/scripts && ./start_server.sh
+fi
+
+exec "$@"
+

tests/milvus-java-test/.gitignore

@@ -0,0 +1,4 @@
+target/
+.idea/
+test-output/
+lib/*

tests/milvus-java-test/README.md

@@ -0,0 +1,29 @@
+# Requirements
+
+- jdk-1.8
+- testng
+
+# How to use this Test Project
+
+1. package and install
+
+```shell
+mvn clean install
+```
+
+2. start or deploy your milvus server
+3. run tests
+
+```shell
+java -cp \"target/MilvusSDkJavaTest-1.0-SNAPSHOT.jar:lib/*\" com.MainClass -h 127.0.0.1
+```
+
+4. get test report
+
+```shell
+firefox test-output/index.html
+```
+
+# Contribution getting started
+
+Add test cases under testng framework

tests/milvus-java-test/ci/function/file_transfer.groovy

@@ -0,0 +1,10 @@
+def FileTransfer (sourceFiles, remoteDirectory, remoteIP, protocol = "ftp", makeEmptyDirs = true) {
+    if (protocol == "ftp") {
+        ftpPublisher masterNodeName: '', paramPublish: [parameterName: ''], alwaysPublishFromMaster: false, continueOnError: false, failOnError: true, publishers: [
+            [configName: "${remoteIP}", transfers: [
+                [asciiMode: false, cleanRemote: false, excludes: '', flatten: false, makeEmptyDirs: "${makeEmptyDirs}", noDefaultExcludes: false, patternSeparator: '[, ]+', remoteDirectory: "${remoteDirectory}", remoteDirectorySDF: false, removePrefix: '', sourceFiles: "${sourceFiles}"]], usePromotionTimestamp: true, useWorkspaceInPromotion: false, verbose: true
+                ]
+            ]
+    }
+}
+return this

tests/milvus-java-test/ci/jenkinsfile/cleanup.groovy

@@ -0,0 +1,13 @@
+try {
+    def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
+    if (!result) {
+        sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
+    }
+} catch (exc) {
+    def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
+    if (!result) {
+        sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
+    }
+    throw exc
+}
+

tests/milvus-java-test/ci/jenkinsfile/deploy_server.groovy

@@ -0,0 +1,16 @@
+try {
+    sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+    sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus'
+    sh 'helm repo update'
+    dir ("milvus-helm") {
+        checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]])
+        dir ("milvus/milvus-gpu") {
+            sh "helm install --wait --timeout 300 --set engine.image.tag=${IMAGE_TAG} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-sdk-test --version 0.3.1 ."
+        }
+    }
+} catch (exc) {
+    echo 'Helm running failed!'
+    sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
+    throw exc
+}
+

tests/milvus-java-test/ci/jenkinsfile/integration_test.groovy

@@ -0,0 +1,13 @@
+timeout(time: 30, unit: 'MINUTES') {
+    try {
+        dir ("milvus-java-test") {
+            sh "mvn clean install"
+            sh "java -cp \"target/MilvusSDkJavaTest-1.0-SNAPSHOT.jar:lib/*\" com.MainClass -h ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-sdk-test.svc.cluster.local"
+        }
+
+    } catch (exc) {
+        echo 'Milvus-SDK-Java Integration Test Failed !'
+        throw exc
+    }
+}
+

tests/milvus-java-test/ci/jenkinsfile/notify.groovy

@@ -0,0 +1,15 @@
+def notify() {
+    if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
+        // Send an email only if the build status has changed from green/unstable to red
+        emailext subject: '$DEFAULT_SUBJECT',
+        body: '$DEFAULT_CONTENT',
+        recipientProviders: [
+            [$class: 'DevelopersRecipientProvider'],
+            [$class: 'RequesterRecipientProvider']
+        ], 
+        replyTo: '$DEFAULT_REPLYTO',
+        to: '$DEFAULT_RECIPIENTS'
+    }
+}
+return this
+

tests/milvus-java-test/ci/jenkinsfile/upload_unit_test_out.groovy

@@ -0,0 +1,13 @@
+timeout(time: 5, unit: 'MINUTES') {
+    dir ("${PROJECT_NAME}_test") {
+        if (fileExists('test_out')) {
+            def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
+            fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
+            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
+                echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
+            }
+        } else {
+            error("Milvus Dev Test Out directory don't exists!")
+        }
+    }
+}

tests/milvus-java-test/ci/main_jenkinsfile

@@ -0,0 +1,110 @@
+pipeline {
+    agent none
+
+    options {
+        timestamps()
+    }
+
+    environment {
+        SRC_BRANCH = "master"
+        IMAGE_TAG = "${params.IMAGE_TAG}-release"
+        HELM_BRANCH = "${params.IMAGE_TAG}" 
+        TEST_URL = "git@192.168.1.105:Test/milvus-java-test.git"
+        TEST_BRANCH = "${params.IMAGE_TAG}"
+    }
+
+    stages {
+        stage("Setup env") {
+            agent {
+                kubernetes {
+                    label 'dev-test'
+                    defaultContainer 'jnlp'
+                    yaml """
+                        apiVersion: v1
+                        kind: Pod
+                        metadata:
+                          labels:
+                            app: milvus
+                            componet: test
+                        spec:
+                          containers:
+                          - name: milvus-testframework-java
+                            image: registry.zilliz.com/milvus/milvus-java-test:v0.1
+                            command:
+                            - cat
+                            tty: true
+                            volumeMounts:
+                            - name: kubeconf
+                              mountPath: /root/.kube/
+                              readOnly: true
+                          volumes:
+                          - name: kubeconf
+                            secret:
+                              secretName: test-cluster-config
+                        """
+                }
+            }
+
+            stages {
+                stage("Deploy Server") {
+                    steps {
+                        gitlabCommitStatus(name: 'Deloy Server') {
+                            container('milvus-testframework-java') {
+                                script {
+                                    load "${env.WORKSPACE}/milvus-java-test/ci/jenkinsfile/deploy_server.groovy"
+                                }
+                            }
+                        }
+                    }
+                }
+                stage("Integration Test") {
+                    steps {
+                        gitlabCommitStatus(name: 'Integration Test') {
+                            container('milvus-testframework-java') {
+                                script {
+                                    print "In integration test stage"
+                                    load "${env.WORKSPACE}/milvus-java-test/ci/jenkinsfile/integration_test.groovy"
+                                }
+                            }
+                        }
+                    }
+                }
+                stage ("Cleanup Env") {
+                    steps {
+                        gitlabCommitStatus(name: 'Cleanup Env') {
+                            container('milvus-testframework-java') {
+                                script {
+                                    load "${env.WORKSPACE}/milvus-java-test/ci/jenkinsfile/cleanup.groovy"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            post {
+                always {
+                    container('milvus-testframework-java') {
+                        script {
+                            load "${env.WORKSPACE}/milvus-java-test/ci/jenkinsfile/cleanup.groovy"
+                        }
+                    }
+                }
+                success {
+                    script {
+                        echo "Milvus java-sdk test success !"
+                    }
+                }
+                aborted {
+                    script {
+                        echo "Milvus java-sdk test aborted !"
+                    }
+                }
+                failure {
+                    script {
+                        echo "Milvus java-sdk test failed !"
+                    }
+                }
+            }
+        }
+    }
+}

tests/milvus-java-test/ci/pod_containers/milvus-testframework.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus
+    componet: testframework-java
+spec:
+  containers:
+  - name: milvus-testframework-java
+    image: maven:3.6.2-jdk-8
+    command:
+    - cat
+    tty: true

tests/milvus-java-test/milvus-java-test.iml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4" />

tests/milvus-java-test/pom.xml

@@ -0,0 +1,137 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>milvus</groupId>
+    <artifactId>MilvusSDkJavaTest</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <build>
+        <plugins>
+            <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-dependency-plugin</artifactId>
+            <executions>
+                <execution>
+                    <id>copy-dependencies</id>
+                    <phase>package</phase>
+                    <goals>
+                        <goal>copy-dependencies</goal>
+                    </goals>
+                    <configuration>
+                        <outputDirectory>lib</outputDirectory>
+                        <overWriteReleases>false</overWriteReleases>
+                        <overWriteSnapshots>true</overWriteSnapshots>
+                    </configuration>
+                </execution>
+            </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <grpc.version>1.23.0</grpc.version><!-- CURRENT_GRPC_VERSION -->
+        <protobuf.version>3.9.0</protobuf.version>
+        <protoc.version>3.9.0</protoc.version>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+<!--    <dependencyManagement>-->
+<!--        <dependencies>-->
+<!--            <dependency>-->
+<!--                <groupId>io.grpc</groupId>-->
+<!--                <artifactId>grpc-bom</artifactId>-->
+<!--                <version>${grpc.version}</version>-->
+<!--                <type>pom</type>-->
+<!--                <scope>import</scope>-->
+<!--            </dependency>-->
+<!--        </dependencies>-->
+<!--    </dependencyManagement>-->
+
+    <repositories>
+        <repository>
+            <id>oss.sonatype.org-snapshot</id>
+            <url>http://oss.sonatype.org/content/repositories/snapshots</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.4</version>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <version>1.3</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.testng</groupId>
+            <artifactId>testng</artifactId>
+            <version>6.10</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.9</version>
+        </dependency>
+
+<!--        <dependency>-->
+<!--            <groupId>io.milvus</groupId>-->
+<!--            <artifactId>milvus-sdk-java</artifactId>-->
+<!--            <version>0.1.0</version>-->
+<!--        </dependency>-->
+
+        <dependency>
+            <groupId>io.milvus</groupId>
+            <artifactId>milvus-sdk-java</artifactId>
+            <version>0.2.0-SNAPSHOT</version>
+        </dependency>
+
+<!--        <dependency>-->
+<!--            <groupId>io.grpc</groupId>-->
+<!--            <artifactId>grpc-netty-sher staded</artifactId>-->
+<!--            <scope>runtime</scope>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>io.grpc</groupId>-->
+<!--            <artifactId>grpc-protobuf</artifactId>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>io.grpc</groupId>-->
+<!--            <artifactId>grpc-stub</artifactId>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>javax.annotation</groupId>-->
+<!--            <artifactId>javax.annotation-api</artifactId>-->
+<!--            <version>1.2</version>-->
+<!--            <scope>provided</scope> &lt;!&ndash; not needed at runtime &ndash;&gt;-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>io.grpc</groupId>-->
+<!--            <artifactId>grpc-testing</artifactId>-->
+<!--            <scope>test</scope>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>com.google.protobuf</groupId>-->
+<!--            <artifactId>protobuf-java-util</artifactId>-->
+<!--            <version>${protobuf.version}</version>-->
+<!--        </dependency>-->
+
+    </dependencies>
+
+</project>

tests/milvus-java-test/src/main/java/com/MainClass.java

@@ -0,0 +1,146 @@
+package com;
+
+import io.milvus.client.*;
+import org.apache.commons.cli.*;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.testng.SkipException;
+import org.testng.TestNG;
+import org.testng.annotations.DataProvider;
+import org.testng.xml.XmlClass;
+import org.testng.xml.XmlSuite;
+import org.testng.xml.XmlTest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MainClass {
+    private static String host = "127.0.0.1";
+    private static String port = "19530";
+    private int index_file_size = 50;
+    public int dimension = 128;
+
+    public static void setHost(String host) {
+        MainClass.host = host;
+    }
+
+    public static void setPort(String port) {
+        MainClass.port = port;
+    }
+
+    @DataProvider(name="DefaultConnectArgs")
+    public static Object[][] defaultConnectArgs(){
+        return new Object[][]{{host, port}};
+    }
+
+    @DataProvider(name="ConnectInstance")
+    public Object[][] connectInstance() throws ConnectFailedException {
+        MilvusClient client = new MilvusGrpcClient();
+        ConnectParam connectParam = new ConnectParam.Builder()
+                .withHost(host)
+                .withPort(port)
+                .build();
+        client.connect(connectParam);
+        String tableName = RandomStringUtils.randomAlphabetic(10);
+        return new Object[][]{{client, tableName}};
+    }
+
+    @DataProvider(name="DisConnectInstance")
+    public Object[][] disConnectInstance() throws ConnectFailedException {
+        // Generate connection instance
+        MilvusClient client = new MilvusGrpcClient();
+        ConnectParam connectParam = new ConnectParam.Builder()
+                .withHost(host)
+                .withPort(port)
+                .build();
+        client.connect(connectParam);
+        try {
+            client.disconnect();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        String tableName = RandomStringUtils.randomAlphabetic(10);
+        return new Object[][]{{client, tableName}};
+    }
+
+    @DataProvider(name="Table")
+    public Object[][] provideTable() throws ConnectFailedException {
+        Object[][] tables = new Object[2][2];
+        MetricType[] metricTypes = { MetricType.L2, MetricType.IP };
+        for (int i = 0; i < metricTypes.length; ++i) {
+            String tableName = metricTypes[i].toString()+"_"+RandomStringUtils.randomAlphabetic(10);
+            // Generate connection instance
+            MilvusClient client = new MilvusGrpcClient();
+            ConnectParam connectParam = new ConnectParam.Builder()
+                    .withHost(host)
+                    .withPort(port)
+                    .build();
+            client.connect(connectParam);
+            TableSchema tableSchema = new TableSchema.Builder(tableName, dimension)
+                    .withIndexFileSize(index_file_size)
+                    .withMetricType(metricTypes[i])
+                    .build();
+            Response res = client.createTable(tableSchema);
+            if (!res.ok()) {
+                System.out.println(res.getMessage());
+                throw new SkipException("Table created failed");
+            }
+            tables[i] = new Object[]{client, tableName};
+        }
+        return tables;
+    }
+
+    public static void main(String[] args) {
+        CommandLineParser parser = new DefaultParser();
+        Options options = new Options();
+        options.addOption("h", "host", true, "milvus-server hostname/ip");
+        options.addOption("p", "port", true, "milvus-server port");
+        try {
+            CommandLine cmd = parser.parse(options, args);
+            String host = cmd.getOptionValue("host");
+            if (host != null) {
+                setHost(host);
+            }
+            String port = cmd.getOptionValue("port");
+            if (port != null) {
+                setPort(port);
+            }
+            System.out.println("Host: "+host+", Port: "+port);
+        }
+        catch(ParseException exp) {
+            System.err.println("Parsing failed.  Reason: " + exp.getMessage() );
+        }
+
+//        TestListenerAdapter tla = new TestListenerAdapter();
+//        TestNG testng = new TestNG();
+//        testng.setTestClasses(new Class[] { TestPing.class });
+//        testng.setTestClasses(new Class[] { TestConnect.class });
+//        testng.addListener(tla);
+//        testng.run();
+
+        XmlSuite suite = new XmlSuite();
+        suite.setName("TmpSuite");
+
+        XmlTest test = new XmlTest(suite);
+        test.setName("TmpTest");
+        List<XmlClass> classes = new ArrayList<XmlClass>();
+
+        classes.add(new XmlClass("com.TestPing"));
+        classes.add(new XmlClass("com.TestAddVectors"));
+        classes.add(new XmlClass("com.TestConnect"));
+        classes.add(new XmlClass("com.TestDeleteVectors"));
+        classes.add(new XmlClass("com.TestIndex"));
+        classes.add(new XmlClass("com.TestSearchVectors"));
+        classes.add(new XmlClass("com.TestTable"));
+        classes.add(new XmlClass("com.TestTableCount"));
+
+        test.setXmlClasses(classes) ;
+
+        List<XmlSuite> suites = new ArrayList<XmlSuite>();
+        suites.add(suite);
+        TestNG tng = new TestNG();
+        tng.setXmlSuites(suites);
+        tng.run();
+
+    }
+
+}

tests/milvus-java-test/src/main/java/com/TestAddVectors.java

@@ -0,0 +1,150 @@
+package com;
+
+import io.milvus.client.InsertParam;
+import io.milvus.client.InsertResponse;
+import io.milvus.client.MilvusClient;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class TestAddVectors {
+    int dimension = 128;
+
+    public List<List<Float>> gen_vectors(Integer nb) {
+        List<List<Float>> xb = new LinkedList<>();
+        Random random = new Random();
+        for (int i = 0; i < nb; ++i) {
+            LinkedList<Float> vector = new LinkedList<>();
+            for (int j = 0; j < dimension; j++) {
+                vector.add(random.nextFloat());
+            }
+            xb.add(vector);
+        }
+        return xb;
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors_table_not_existed(MilvusClient client, String tableName) throws InterruptedException {
+        int nb = 10000;
+        List<List<Float>> vectors = gen_vectors(nb);
+        String tableNameNew = tableName + "_";
+        InsertParam insertParam = new InsertParam.Builder(tableNameNew, vectors).build();
+        InsertResponse res = client.insert(insertParam);
+        assert(!res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_add_vectors_without_connect(MilvusClient client, String tableName) throws InterruptedException {
+        int nb = 100;
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        InsertResponse res = client.insert(insertParam);
+        assert(!res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors(MilvusClient client, String tableName) throws InterruptedException {
+        int nb = 10000;
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        InsertResponse res = client.insert(insertParam);
+        assert(res.getResponse().ok());
+        Thread.currentThread().sleep(1000);
+        // Assert table row count
+        Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb);
+    }
+
+//    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+//    public void test_add_vectors_timeout(MilvusClient client, String tableName) throws InterruptedException {
+//        int nb = 200000;
+//        List<List<Float>> vectors = gen_vectors(nb);
+//        System.out.println(new Date());
+//        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withTimeout(1).build();
+//        InsertResponse res = client.insert(insertParam);
+//        assert(!res.getResponse().ok());
+//    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors_big_data(MilvusClient client, String tableName) throws InterruptedException {
+        int nb = 500000;
+        List<List<Float>> vectors = gen_vectors(nb);
+        System.out.println(new Date());
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        InsertResponse res = client.insert(insertParam);
+        assert(res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors_with_ids(MilvusClient client, String tableName) throws InterruptedException {
+        int nb = 10000;
+        List<List<Float>> vectors = gen_vectors(nb);
+        // Add vectors with ids
+        List<Long> vectorIds;
+        vectorIds = Stream.iterate(0L, n -> n)
+                .limit(nb)
+                .collect(Collectors.toList());
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withVectorIds(vectorIds).build();
+        InsertResponse res = client.insert(insertParam);
+        assert(res.getResponse().ok());
+        Thread.currentThread().sleep(2000);
+        // Assert table row count
+        Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb);
+    }
+
+    // TODO: MS-628
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors_with_invalid_ids(MilvusClient client, String tableName) {
+        int nb = 10;
+        List<List<Float>> vectors = gen_vectors(nb);
+        // Add vectors with ids
+        List<Long> vectorIds;
+        vectorIds = Stream.iterate(0L, n -> n)
+                .limit(nb+1)
+                .collect(Collectors.toList());
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withVectorIds(vectorIds).build();
+        InsertResponse res = client.insert(insertParam);
+        assert(!res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors_with_invalid_dimension(MilvusClient client, String tableName) {
+        int nb = 10000;
+        List<List<Float>> vectors = gen_vectors(nb);
+        vectors.get(0).add((float) 0);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        InsertResponse res = client.insert(insertParam);
+        assert(!res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors_with_invalid_vectors(MilvusClient client, String tableName) {
+        int nb = 10000;
+        List<List<Float>> vectors = gen_vectors(nb);
+        vectors.set(0, new ArrayList<>());
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        InsertResponse res = client.insert(insertParam);
+        assert(!res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors_repeatably(MilvusClient client, String tableName) throws InterruptedException {
+        int nb = 100000;
+        int loops = 10;
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        InsertResponse res = null;
+        for (int i = 0; i < loops; ++i ) {
+            long startTime = System.currentTimeMillis();
+            res = client.insert(insertParam);
+            long endTime = System.currentTimeMillis();
+            System.out.println("Total execution time: " + (endTime-startTime) + "ms");
+        }
+        Thread.currentThread().sleep(1000);
+        // Assert table row count
+        Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb * loops);
+    }
+
+}

tests/milvus-java-test/src/main/java/com/TestConnect.java

@@ -0,0 +1,89 @@
+package com;
+
+import io.milvus.client.*;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class TestConnect {
+    @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class)
+    public void test_connect(String host, String port) throws ConnectFailedException {
+        System.out.println("Host: "+host+", Port: "+port);
+        MilvusClient client = new MilvusGrpcClient();
+        ConnectParam connectParam = new ConnectParam.Builder()
+                .withHost(host)
+                .withPort(port)
+                .build();
+        Response res = client.connect(connectParam);
+        assert(res.ok());
+        assert(client.isConnected());
+    }
+
+    @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class)
+    public void test_connect_repeat(String host, String port) {
+        MilvusGrpcClient client = new MilvusGrpcClient();
+        ConnectParam connectParam = new ConnectParam.Builder()
+                .withHost(host)
+                .withPort(port)
+                .build();
+        Response res = null;
+        try {
+            res = client.connect(connectParam);
+            res = client.connect(connectParam);
+        } catch (ConnectFailedException e) {
+            e.printStackTrace();
+        }
+        assert (res.ok());
+        assert(client.isConnected());
+    }
+
+    @Test(dataProvider="InvalidConnectArgs")
+    public void test_connect_invalid_connect_args(String ip, String port) {
+        MilvusClient client = new MilvusGrpcClient();
+        ConnectParam connectParam = new ConnectParam.Builder()
+                .withHost(ip)
+                .withPort(port)
+                .build();
+        Response res = null;
+        try {
+            res = client.connect(connectParam);
+        } catch (ConnectFailedException e) {
+            e.printStackTrace();
+        }
+        Assert.assertEquals(res, null);
+        assert(!client.isConnected());
+    }
+
+    // TODO: MS-615
+    @DataProvider(name="InvalidConnectArgs")
+    public Object[][] generate_invalid_connect_args() {
+        String port = "19530";
+        String ip = "";
+        return new Object[][]{
+                {"1.1.1.1", port},
+                {"255.255.0.0", port},
+                {"1.2.2", port},
+                {"中文", port},
+                {"www.baidu.com", "100000"},
+                {"127.0.0.1", "100000"},
+                {"www.baidu.com", "80"},
+        };
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_disconnect(MilvusClient client, String tableName){
+        assert(!client.isConnected());
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_disconnect_repeatably(MilvusClient client, String tableName){
+        Response res = null;
+        try {
+            res = client.disconnect();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        assert(!res.ok());
+        assert(!client.isConnected());
+    }
+}

tests/milvus-java-test/src/main/java/com/TestDeleteVectors.java

@@ -0,0 +1,116 @@
+package com;
+
+import java.util.*;
+
+public class TestDeleteVectors {
+    int index_file_size = 50;
+    int dimension = 128;
+
+    public List<List<Float>> gen_vectors(Integer nb) {
+        List<List<Float>> xb = new LinkedList<>();
+        Random random = new Random();
+        for (int i = 0; i < nb; ++i) {
+            LinkedList<Float> vector = new LinkedList<>();
+            for (int j = 0; j < dimension; j++) {
+                vector.add(random.nextFloat());
+            }
+            xb.add(vector);
+        }
+        return xb;
+    }
+
+    public static Date getDeltaDate(int delta) {
+        Date today = new Date();
+        Calendar c = Calendar.getInstance();
+        c.setTime(today);
+        c.add(Calendar.DAY_OF_MONTH, delta);
+        return c.getTime();
+    }
+
+//    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+//    public void test_delete_vectors(MilvusClient client, String tableName) throws InterruptedException {
+//        int nb = 10000;
+//        List<List<Float>> vectors = gen_vectors(nb);
+//        // Add vectors
+//        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+//        InsertResponse res = client.insert(insertParam);
+//        assert(res.getResponse().ok());
+//        Thread.sleep(1000);
+//        DateRange dateRange = new DateRange(getDeltaDate(-1), getDeltaDate(1));
+//        DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableName).build();
+//        Response res_delete = client.deleteByRange(param);
+//        assert(res_delete.ok());
+//        Thread.sleep(1000);
+//        // Assert table row count
+//        Assert.assertEquals(client.getTableRowCount(tableParam).getTableRowCount(), 0);
+//    }
+
+//    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+//    public void test_delete_vectors_table_not_existed(MilvusClient client, String tableName) throws InterruptedException {
+//        String tableNameNew = tableName + "_";
+//        DateRange dateRange = new DateRange(getDeltaDate(-1), getDeltaDate(1));
+//        DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableNameNew).build();
+//        Response res_delete = client.deleteByRange(param);
+//        assert(!res_delete.ok());
+//    }
+
+//    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+//    public void test_delete_vectors_without_connect(MilvusClient client, String tableName) throws InterruptedException {
+//        DateRange dateRange = new DateRange(getDeltaDate(-1), getDeltaDate(1));
+//        DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableName).build();
+//        Response res_delete = client.deleteByRange(param);
+//        assert(!res_delete.ok());
+//    }
+//
+//    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+//    public void test_delete_vectors_table_empty(MilvusClient client, String tableName) throws InterruptedException {
+//        DateRange dateRange = new DateRange(getDeltaDate(-1), getDeltaDate(1));
+//        DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableName).build();
+//        Response res_delete = client.deleteByRange(param);
+//        assert(res_delete.ok());
+//    }
+
+//    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+//    public void test_delete_vectors_invalid_date_range(MilvusClient client, String tableName) throws InterruptedException {
+//        int nb = 100;
+//        List<List<Float>> vectors = gen_vectors(nb);
+//        // Add vectors
+//        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+//        InsertResponse res = client.insert(insertParam);
+//        assert(res.getResponse().ok());
+//        Thread.sleep(1000);
+//        DateRange dateRange = new DateRange(getDeltaDate(1), getDeltaDate(0));
+//        DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableName).build();
+//        Response res_delete = client.deleteByRange(param);
+//        assert(!res_delete.ok());
+//    }
+
+//    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+//    public void test_delete_vectors_invalid_date_range_1(MilvusClient client, String tableName) throws InterruptedException {
+//        int nb = 100;
+//        List<List<Float>> vectors = gen_vectors(nb);
+//        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+//        InsertResponse res = client.insert(insertParam);
+//        assert(res.getResponse().ok());
+//        DateRange dateRange = new DateRange(getDeltaDate(2), getDeltaDate(-1));
+//        DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableName).build();
+//        Response res_delete = client.deleteByRange(param);
+//        assert(!res_delete.ok());
+//    }
+
+//    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+//    public void test_delete_vectors_no_result(MilvusClient client, String tableName) throws InterruptedException {
+//        int nb = 100;
+//        List<List<Float>> vectors = gen_vectors(nb);
+//        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+//        InsertResponse res = client.insert(insertParam);
+//        assert(res.getResponse().ok());
+//        Thread.sleep(1000);
+//        DateRange dateRange = new DateRange(getDeltaDate(-3), getDeltaDate(-2));
+//        DeleteByRangeParam param = new DeleteByRangeParam.Builder(dateRange, tableName).build();
+//        Response res_delete = client.deleteByRange(param);
+//        assert(res_delete.ok());
+//        Assert.assertEquals(client.getTableRowCount(tableParam).getTableRowCount(), nb);
+//    }
+
+}

tests/milvus-java-test/src/main/java/com/TestIndex.java

@@ -0,0 +1,324 @@
+package com;
+
+import io.milvus.client.*;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+
+public class TestIndex {
+    int index_file_size = 10;
+    int dimension = 128;
+    int n_list = 1024;
+    int default_n_list = 16384;
+    int nb = 100000;
+    IndexType indexType = IndexType.IVF_SQ8;
+    IndexType defaultIndexType = IndexType.FLAT;
+
+    public List<List<Float>> gen_vectors(Integer nb) {
+        List<List<Float>> xb = new LinkedList<>();
+        Random random = new Random();
+        for (int i = 0; i < nb; ++i) {
+            LinkedList<Float> vector = new LinkedList<>();
+            for (int j = 0; j < dimension; j++) {
+                vector.add(random.nextFloat());
+            }
+            xb.add(vector);
+        }
+        return xb;
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_create_index(MilvusClient client, String tableName) throws InterruptedException {
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(res_create.ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_create_index_repeatably(MilvusClient client, String tableName) throws InterruptedException {
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        res_create = client.createIndex(createIndexParam);
+        assert(res_create.ok());
+        DescribeIndexResponse res = client.describeIndex(tableName);
+        assert(res.getResponse().ok());
+        Index index1 = res.getIndex().get();
+        Assert.assertEquals(index1.getNList(), n_list);
+        Assert.assertEquals(index1.getIndexType(), indexType);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_create_index_FLAT(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.FLAT;
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(res_create.ok());
+        DescribeIndexResponse res = client.describeIndex(tableName);
+        assert(res.getResponse().ok());
+        Index index1 = res.getIndex().get();
+        Assert.assertEquals(index1.getIndexType(), indexType);
+    }
+
+//    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+//    public void test_create_index_FLAT_timeout(MilvusClient client, String tableName) throws InterruptedException {
+//        int nb = 500000;
+//        IndexType indexType = IndexType.IVF_SQ8;
+//        List<List<Float>> vectors = gen_vectors(nb);
+//        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+//        client.insert(insertParam);
+//        Index index = new Index.Builder().withIndexType(indexType)
+//                .withNList(n_list)
+//                .build();
+//        System.out.println(new Date());
+//        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).withTimeout(1).build();
+//        Response res_create = client.createIndex(createIndexParam);
+//        assert(!res_create.ok());
+//    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_create_index_IVFLAT(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVFLAT;
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(res_create.ok());
+        DescribeIndexResponse res = client.describeIndex(tableName);
+        assert(res.getResponse().ok());
+        Index index1 = res.getIndex().get();
+        Assert.assertEquals(index1.getIndexType(), indexType);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_create_index_IVFSQ8(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVF_SQ8;
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(res_create.ok());
+        DescribeIndexResponse res = client.describeIndex(tableName);
+        assert(res.getResponse().ok());
+        Index index1 = res.getIndex().get();
+        Assert.assertEquals(index1.getIndexType(), indexType);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_create_index_IVFSQ8H(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVF_SQ8H;
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(res_create.ok());
+        DescribeIndexResponse res = client.describeIndex(tableName);
+        assert(res.getResponse().ok());
+        Index index1 = res.getIndex().get();
+        Assert.assertEquals(index1.getIndexType(), indexType);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_create_index_with_no_vector(MilvusClient client, String tableName) {
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(res_create.ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_create_index_table_not_existed(MilvusClient client, String tableName) throws InterruptedException {
+        String tableNameNew = tableName + "_";
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableNameNew).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(!res_create.ok());
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_create_index_without_connect(MilvusClient client, String tableName) throws InterruptedException {
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(!res_create.ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_create_index_invalid_n_list(MilvusClient client, String tableName) throws InterruptedException {
+        int n_list = 0;
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(!res_create.ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_describe_index(MilvusClient client, String tableName) throws InterruptedException {
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(res_create.ok());
+        DescribeIndexResponse res = client.describeIndex(tableName);
+        assert(res.getResponse().ok());
+        Index index1 = res.getIndex().get();
+        Assert.assertEquals(index1.getNList(), n_list);
+        Assert.assertEquals(index1.getIndexType(), indexType);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_alter_index(MilvusClient client, String tableName) throws InterruptedException {
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(res_create.ok());
+        // Create another index
+        IndexType indexTypeNew = IndexType.IVFLAT;
+        int n_list_new = n_list + 1;
+        Index index_new = new Index.Builder().withIndexType(indexTypeNew)
+                .withNList(n_list_new)
+                .build();
+        CreateIndexParam createIndexParamNew = new CreateIndexParam.Builder(tableName).withIndex(index_new).build();
+        Response res_create_new = client.createIndex(createIndexParamNew);
+        assert(res_create_new.ok());
+        DescribeIndexResponse res = client.describeIndex(tableName);
+        assert(res_create.ok());
+        Index index1 = res.getIndex().get();
+        Assert.assertEquals(index1.getNList(), n_list_new);
+        Assert.assertEquals(index1.getIndexType(), indexTypeNew);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_describe_index_table_not_existed(MilvusClient client, String tableName) throws InterruptedException {
+        String tableNameNew = tableName + "_";
+        DescribeIndexResponse res = client.describeIndex(tableNameNew);
+        assert(!res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_describe_index_without_connect(MilvusClient client, String tableName) throws InterruptedException {
+        DescribeIndexResponse res = client.describeIndex(tableName);
+        assert(!res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_drop_index(MilvusClient client, String tableName) throws InterruptedException {
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(defaultIndexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(res_create.ok());
+        Response res_drop = client.dropIndex(tableName);
+        assert(res_drop.ok());
+        DescribeIndexResponse res = client.describeIndex(tableName);
+        assert(res.getResponse().ok());
+        Index index1 = res.getIndex().get();
+        Assert.assertEquals(index1.getNList(), default_n_list);
+        Assert.assertEquals(index1.getIndexType(), defaultIndexType);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_drop_index_repeatably(MilvusClient client, String tableName) throws InterruptedException {
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(defaultIndexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        Response res_create = client.createIndex(createIndexParam);
+        assert(res_create.ok());
+        Response res_drop = client.dropIndex(tableName);
+        res_drop = client.dropIndex(tableName);
+        assert(res_drop.ok());
+        DescribeIndexResponse res = client.describeIndex(tableName);
+        assert(res.getResponse().ok());
+        Index index1 = res.getIndex().get();
+        Assert.assertEquals(index1.getNList(), default_n_list);
+        Assert.assertEquals(index1.getIndexType(), defaultIndexType);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_drop_index_table_not_existed(MilvusClient client, String tableName) throws InterruptedException {
+        String tableNameNew = tableName + "_";
+        Response res_drop = client.dropIndex(tableNameNew);
+        assert(!res_drop.ok());
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_drop_index_without_connect(MilvusClient client, String tableName) throws InterruptedException {
+        Response res_drop = client.dropIndex(tableName);
+        assert(!res_drop.ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_drop_index_no_index_created(MilvusClient client, String tableName) throws InterruptedException {
+        List<List<Float>> vectors = gen_vectors(nb);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Response res_drop = client.dropIndex(tableName);
+        assert(res_drop.ok());
+        DescribeIndexResponse res = client.describeIndex(tableName);
+        assert(res.getResponse().ok());
+        Index index1 = res.getIndex().get();
+        Assert.assertEquals(index1.getNList(), default_n_list);
+        Assert.assertEquals(index1.getIndexType(), defaultIndexType);
+    }
+
+}

tests/milvus-java-test/src/main/java/com/TestMix.java

@@ -0,0 +1,225 @@
+package com;
+
+import io.milvus.client.*;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+public class TestMix {
+    private int dimension = 128;
+    private int nb = 100000;
+    int n_list = 8192;
+    int n_probe = 20;
+    int top_k = 10;
+    double epsilon = 0.001;
+    int index_file_size = 20;
+
+    public List<Float> normalize(List<Float> w2v){
+        float squareSum = w2v.stream().map(x -> x * x).reduce((float) 0, Float::sum);
+        final float norm = (float) Math.sqrt(squareSum);
+        w2v = w2v.stream().map(x -> x / norm).collect(Collectors.toList());
+        return w2v;
+    }
+
+    public List<List<Float>> gen_vectors(int nb, boolean norm) {
+        List<List<Float>> xb = new ArrayList<>();
+        Random random = new Random();
+        for (int i = 0; i < nb; ++i) {
+            List<Float> vector = new ArrayList<>();
+            for (int j = 0; j < dimension; j++) {
+                vector.add(random.nextFloat());
+            }
+            if (norm == true) {
+                vector = normalize(vector);
+            }
+            xb.add(vector);
+        }
+        return xb;
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_vectors_threads(MilvusClient client, String tableName) throws InterruptedException {
+        int thread_num = 10;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(IndexType.IVF_SQ8)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        ForkJoinPool executor = new ForkJoinPool();
+        for (int i = 0; i < thread_num; i++) {
+            executor.execute(
+                    () -> {
+                        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+                        SearchResponse res_search = client.search(searchParam);
+                        assert (res_search.getResponse().ok());
+                    });
+        }
+        executor.awaitQuiescence(100, TimeUnit.SECONDS);
+        executor.shutdown();
+    }
+
+    @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class)
+    public void test_connect_threads(String host, String port) throws ConnectFailedException {
+        int thread_num = 100;
+        ForkJoinPool executor = new ForkJoinPool();
+        for (int i = 0; i < thread_num; i++) {
+            executor.execute(
+                    () -> {
+                        MilvusClient client = new MilvusGrpcClient();
+                        ConnectParam connectParam = new ConnectParam.Builder()
+                                .withHost(host)
+                                .withPort(port)
+                                .build();
+                        try {
+                            client.connect(connectParam);
+                        } catch (ConnectFailedException e) {
+                            e.printStackTrace();
+                        }
+                        assert(client.isConnected());
+                        try {
+                            client.disconnect();
+                        } catch (InterruptedException e) {
+                            e.printStackTrace();
+                        }
+                    });
+        }
+        executor.awaitQuiescence(100, TimeUnit.SECONDS);
+        executor.shutdown();
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_vectors_threads(MilvusClient client, String tableName) throws InterruptedException {
+        int thread_num = 10;
+        List<List<Float>> vectors = gen_vectors(nb,false);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        ForkJoinPool executor = new ForkJoinPool();
+        for (int i = 0; i < thread_num; i++) {
+            executor.execute(
+                    () -> {
+                        InsertResponse res_insert = client.insert(insertParam);
+                        assert (res_insert.getResponse().ok());
+                    });
+        }
+        executor.awaitQuiescence(100, TimeUnit.SECONDS);
+        executor.shutdown();
+
+        Thread.sleep(2000);
+        GetTableRowCountResponse getTableRowCountResponse = client.getTableRowCount(tableName);
+        Assert.assertEquals(getTableRowCountResponse.getTableRowCount(), thread_num * nb);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_index_vectors_threads(MilvusClient client, String tableName) throws InterruptedException {
+        int thread_num = 50;
+        List<List<Float>> vectors = gen_vectors(nb,false);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        ForkJoinPool executor = new ForkJoinPool();
+        for (int i = 0; i < thread_num; i++) {
+            executor.execute(
+                    () -> {
+                        InsertResponse res_insert = client.insert(insertParam);
+                        Index index = new Index.Builder().withIndexType(IndexType.IVF_SQ8)
+                                .withNList(n_list)
+                                .build();
+                        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+                        client.createIndex(createIndexParam);
+                        assert (res_insert.getResponse().ok());
+                    });
+        }
+        executor.awaitQuiescence(300, TimeUnit.SECONDS);
+        executor.shutdown();
+        Thread.sleep(2000);
+        GetTableRowCountResponse getTableRowCountResponse = client.getTableRowCount(tableName);
+        Assert.assertEquals(getTableRowCountResponse.getTableRowCount(), thread_num * nb);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_add_search_vectors_threads(MilvusClient client, String tableName) throws InterruptedException {
+        int thread_num = 50;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, true);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        ForkJoinPool executor = new ForkJoinPool();
+        for (int i = 0; i < thread_num; i++) {
+            executor.execute(
+                    () -> {
+                        InsertResponse res_insert = client.insert(insertParam);
+                        assert (res_insert.getResponse().ok());
+                        try {
+                            TimeUnit.SECONDS.sleep(1);
+                        } catch (InterruptedException e) {
+                            e.printStackTrace();
+                        }
+                        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+                        SearchResponse res_search = client.search(searchParam);
+                        assert (res_search.getResponse().ok());
+                        List<List<SearchResponse.QueryResult>> res = client.search(searchParam).getQueryResultsList();
+                        double distance = res.get(0).get(0).getDistance();
+                        if (tableName.startsWith("L2")) {
+                            Assert.assertEquals(distance, 0.0, epsilon);
+                        }else if (tableName.startsWith("IP")) {
+                            Assert.assertEquals(distance, 1.0, epsilon);
+                        }
+                    });
+        }
+        executor.awaitQuiescence(300, TimeUnit.SECONDS);
+        executor.shutdown();
+        Thread.sleep(2000);
+        GetTableRowCountResponse getTableRowCountResponse = client.getTableRowCount(tableName);
+        Assert.assertEquals(getTableRowCountResponse.getTableRowCount(), thread_num * nb);
+    }
+
+    @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class)
+    public void test_create_insert_delete_threads(String host, String port) {
+        int thread_num = 100;
+        List<List<Float>> vectors = gen_vectors(nb,false);
+        ForkJoinPool executor = new ForkJoinPool();
+        for (int i = 0; i < thread_num; i++) {
+            executor.execute(
+                    () -> {
+                        MilvusClient client = new MilvusGrpcClient();
+                        ConnectParam connectParam = new ConnectParam.Builder()
+                                .withHost(host)
+                                .withPort(port)
+                                .build();
+                        try {
+                            client.connect(connectParam);
+                        } catch (ConnectFailedException e) {
+                            e.printStackTrace();
+                        }
+                        assert(client.isConnected());
+                        String tableName = RandomStringUtils.randomAlphabetic(10);
+                        TableSchema tableSchema = new TableSchema.Builder(tableName, dimension)
+                                                                 .withIndexFileSize(index_file_size)
+                                                                 .withMetricType(MetricType.IP)
+                                                                 .build();
+                        client.createTable(tableSchema);
+                        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+                        client.insert(insertParam);
+                        Response response = client.dropTable(tableName);
+                        Assert.assertTrue(response.ok());
+                        try {
+                            client.disconnect();
+                        } catch (InterruptedException e) {
+                            e.printStackTrace();
+                        }
+                    });
+        }
+        executor.awaitQuiescence(100, TimeUnit.SECONDS);
+        executor.shutdown();
+    }
+
+}

tests/milvus-java-test/src/main/java/com/TestPing.java

@@ -0,0 +1,25 @@
+package com;
+
+import io.milvus.client.*;
+import org.testng.annotations.Test;
+
+public class TestPing {
+    @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class)
+    public void test_server_status(String host, String port) throws ConnectFailedException {
+        System.out.println("Host: "+host+", Port: "+port);
+        MilvusClient client = new MilvusGrpcClient();
+        ConnectParam connectParam = new ConnectParam.Builder()
+                .withHost(host)
+                .withPort(port)
+                .build();
+        client.connect(connectParam);
+        Response res = client.getServerStatus();
+        assert (res.ok());
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_server_status_without_connected(MilvusGrpcClient client, String tableName){
+        Response res = client.getServerStatus();
+        assert (!res.ok());
+    }
+}

tests/milvus-java-test/src/main/java/com/TestSearchVectors.java

@@ -0,0 +1,470 @@
+package com;
+
+import io.milvus.client.*;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class TestSearchVectors {
+    int index_file_size = 10;
+    int dimension = 128;
+    int n_list = 1024;
+    int default_n_list = 16384;
+    int nb = 100000;
+    int n_probe = 20;
+    int top_k = 10;
+    double epsilon = 0.001;
+    IndexType indexType = IndexType.IVF_SQ8;
+    IndexType defaultIndexType = IndexType.FLAT;
+
+
+    public List<Float> normalize(List<Float> w2v){
+        float squareSum = w2v.stream().map(x -> x * x).reduce((float) 0, Float::sum);
+        final float norm = (float) Math.sqrt(squareSum);
+        w2v = w2v.stream().map(x -> x / norm).collect(Collectors.toList());
+        return w2v;
+    }
+
+    public List<List<Float>> gen_vectors(int nb, boolean norm) {
+        List<List<Float>> xb = new ArrayList<>();
+        Random random = new Random();
+        for (int i = 0; i < nb; ++i) {
+            List<Float> vector = new ArrayList<>();
+            for (int j = 0; j < dimension; j++) {
+                vector.add(random.nextFloat());
+            }
+            if (norm == true) {
+                vector = normalize(vector);
+            }
+            xb.add(vector);
+        }
+        return xb;
+    }
+
+    public static Date getDeltaDate(int delta) {
+        Date today = new Date();
+        Calendar c = Calendar.getInstance();
+        c.setTime(today);
+        c.add(Calendar.DAY_OF_MONTH, delta);
+        return c.getTime();
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_table_not_existed(MilvusClient client, String tableName) throws InterruptedException {
+        String tableNameNew = tableName + "_";
+        int nq = 5;
+        int nb = 100;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        SearchParam searchParam = new SearchParam.Builder(tableNameNew, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        SearchResponse res_search = client.search(searchParam);
+        assert (!res_search.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_index_IVFLAT(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVFLAT;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        List<List<SearchResponse.QueryResult>> res_search = client.search(searchParam).getQueryResultsList();
+        Assert.assertEquals(res_search.size(), nq);
+        Assert.assertEquals(res_search.get(0).size(), top_k);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_ids_IVFLAT(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVFLAT;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, true);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        List<Long> vectorIds;
+        vectorIds = Stream.iterate(0L, n -> n)
+                .limit(nb)
+                .collect(Collectors.toList());
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withVectorIds(vectorIds).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        List<List<SearchResponse.QueryResult>> res_search = client.search(searchParam).getQueryResultsList();
+        Assert.assertEquals(res_search.get(0).get(0).getVectorId(), 0L);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_IVFLAT(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVFLAT;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Thread.sleep(2000);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        List<List<SearchResponse.QueryResult>> res_search = client.search(searchParam).getQueryResultsList();
+        Assert.assertEquals(res_search.size(), nq);
+        Assert.assertEquals(res_search.get(0).size(), top_k);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_distance_IVFLAT(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVFLAT;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, true);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        List<List<SearchResponse.QueryResult>> res_search = client.search(searchParam).getQueryResultsList();
+        double distance = res_search.get(0).get(0).getDistance();
+        if (tableName.startsWith("L2")) {
+            Assert.assertEquals(distance, 0.0, epsilon);
+        }else if (tableName.startsWith("IP")) {
+            Assert.assertEquals(distance, 1.0, epsilon);
+        }
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_index_IVFSQ8(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVF_SQ8;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        List<List<SearchResponse.QueryResult>> res_search = client.search(searchParam).getQueryResultsList();
+        Assert.assertEquals(res_search.size(), nq);
+        Assert.assertEquals(res_search.get(0).size(), top_k);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_IVFSQ8(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVF_SQ8;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Thread.sleep(1000);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        List<List<SearchResponse.QueryResult>> res_search = client.search(searchParam).getQueryResultsList();
+        Assert.assertEquals(res_search.size(), nq);
+        Assert.assertEquals(res_search.get(0).size(), top_k);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_distance_IVFSQ8(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVF_SQ8;
+        int nq = 5;
+        int nb = 1000;
+        List<List<Float>> vectors = gen_vectors(nb, true);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(default_n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        List<List<Double>> res_search = client.search(searchParam).getResultDistancesList();
+        for (int i = 0; i < nq; i++) {
+            double distance = res_search.get(i).get(0);
+            System.out.println(distance);
+            if (tableName.startsWith("L2")) {
+                Assert.assertEquals(distance, 0.0, epsilon);
+            }else if (tableName.startsWith("IP")) {
+                Assert.assertEquals(distance, 1.0, epsilon);
+            }
+        }
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_index_FLAT(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.FLAT;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        List<List<SearchResponse.QueryResult>> res_search = client.search(searchParam).getQueryResultsList();
+        Assert.assertEquals(res_search.size(), nq);
+        Assert.assertEquals(res_search.get(0).size(), top_k);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_FLAT(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.FLAT;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Thread.sleep(1000);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        List<List<SearchResponse.QueryResult>> res_search = client.search(searchParam).getQueryResultsList();
+        Assert.assertEquals(res_search.size(), nq);
+        Assert.assertEquals(res_search.get(0).size(), top_k);
+    }
+
+//    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+//    public void test_search_FLAT_timeout(MilvusClient client, String tableName) throws InterruptedException {
+//        IndexType indexType = IndexType.FLAT;
+//        int nb = 100000;
+//        int nq = 1000;
+//        int top_k = 2048;
+//        List<List<Float>> vectors = gen_vectors(nb, false);
+//        List<List<Float>> vectors = gen_vectors(nb, false);
+//        List<List<Float>> queryVectors = vectors.subList(0,nq);
+//        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+//        client.insert(insertParam);
+//        Thread.sleep(1000);
+//        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withTimeout(1).build();
+//        System.out.println(new Date());
+//        SearchResponse res_search = client.search(searchParam);
+//        assert (!res_search.getResponse().ok());
+//    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_FLAT_big_data_size(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.FLAT;
+        int nb = 100000;
+        int nq = 2000;
+        int top_k = 2048;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Thread.sleep(1000);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        System.out.println(new Date());
+        SearchResponse res_search = client.search(searchParam);
+        assert (res_search.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_distance_FLAT(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.FLAT;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, true);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build();
+        List<List<SearchResponse.QueryResult>> res_search = client.search(searchParam).getQueryResultsList();
+        double distance = res_search.get(0).get(0).getDistance();
+        if (tableName.startsWith("L2")) {
+            Assert.assertEquals(distance, 0.0, epsilon);
+        }else if (tableName.startsWith("IP")) {
+            Assert.assertEquals(distance, 1.0, epsilon);
+        }
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_invalid_n_probe(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVF_SQ8;
+        int nq = 5;
+        int n_probe_new = 0;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe_new).withTopK(top_k).build();
+        SearchResponse res_search = client.search(searchParam);
+        assert (!res_search.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_invalid_top_k(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVF_SQ8;
+        int nq = 5;
+        int top_k_new = 0;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k_new).build();
+        SearchResponse res_search = client.search(searchParam);
+        assert (!res_search.getResponse().ok());
+    }
+
+//    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+//    public void test_search_invalid_query_vectors(MilvusClient client, String tableName) throws InterruptedException {
+//        IndexType indexType = IndexType.IVF_SQ8;
+//        int nq = 5;
+//        List<List<Float>> vectors = gen_vectors(nb, false);
+//        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+//        client.insert(insertParam);
+//        Index index = new Index.Builder().withIndexType(indexType)
+//                .withNList(n_list)
+//                .build();
+//        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+//        client.createIndex(createIndexParam);
+//        TableParam tableParam = new TableParam.Builder(tableName).build();
+//        SearchParam searchParam = new SearchParam.Builder(tableName, null).withNProbe(n_probe).withTopK(top_k).build();
+//        SearchResponse res_search = client.search(searchParam);
+//        assert (!res_search.getResponse().ok());
+//    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_index_range(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVF_SQ8;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        List<DateRange> dateRange = new ArrayList<>();
+        dateRange.add(new DateRange(getDeltaDate(-1), getDeltaDate(1)));
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withDateRanges(dateRange).build();
+        SearchResponse res_search = client.search(searchParam);
+        assert (res_search.getResponse().ok());
+        List<List<SearchResponse.QueryResult>> res = client.search(searchParam).getQueryResultsList();
+        Assert.assertEquals(res.size(), nq);
+        Assert.assertEquals(res.get(0).size(), top_k);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_range(MilvusClient client, String tableName) throws InterruptedException {
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        List<DateRange> dateRange = new ArrayList<>();
+        dateRange.add(new DateRange(getDeltaDate(-1), getDeltaDate(1)));
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Thread.sleep(1000);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withDateRanges(dateRange).build();
+        SearchResponse res_search = client.search(searchParam);
+        assert (res_search.getResponse().ok());
+        List<List<SearchResponse.QueryResult>> res = client.search(searchParam).getQueryResultsList();
+        Assert.assertEquals(res.size(), nq);
+        Assert.assertEquals(res.get(0).size(), top_k);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_index_range_no_result(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVF_SQ8;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        List<DateRange> dateRange = new ArrayList<>();
+        dateRange.add(new DateRange(getDeltaDate(-3), getDeltaDate(-1)));
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withDateRanges(dateRange).build();
+        SearchResponse res_search = client.search(searchParam);
+        assert (res_search.getResponse().ok());
+        List<List<SearchResponse.QueryResult>> res = client.search(searchParam).getQueryResultsList();
+        Assert.assertEquals(res.size(), 0);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_range_no_result(MilvusClient client, String tableName) throws InterruptedException {
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        List<DateRange> dateRange = new ArrayList<>();
+        dateRange.add(new DateRange(getDeltaDate(-3), getDeltaDate(-1)));
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Thread.sleep(1000);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withDateRanges(dateRange).build();
+        SearchResponse res_search = client.search(searchParam);
+        assert (res_search.getResponse().ok());
+        List<List<SearchResponse.QueryResult>> res = client.search(searchParam).getQueryResultsList();
+        Assert.assertEquals(res.size(), 0);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_index_range_invalid(MilvusClient client, String tableName) throws InterruptedException {
+        IndexType indexType = IndexType.IVF_SQ8;
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        List<DateRange> dateRange = new ArrayList<>();
+        dateRange.add(new DateRange(getDeltaDate(2), getDeltaDate(-1)));
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Index index = new Index.Builder().withIndexType(indexType)
+                .withNList(n_list)
+                .build();
+        CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build();
+        client.createIndex(createIndexParam);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withDateRanges(dateRange).build();
+        SearchResponse res_search = client.search(searchParam);
+        assert (!res_search.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_search_range_invalid(MilvusClient client, String tableName) throws InterruptedException {
+        int nq = 5;
+        List<List<Float>> vectors = gen_vectors(nb, false);
+        List<List<Float>> queryVectors = vectors.subList(0,nq);
+        List<DateRange> dateRange = new ArrayList<>();
+        dateRange.add(new DateRange(getDeltaDate(2), getDeltaDate(-1)));
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();
+        client.insert(insertParam);
+        Thread.sleep(1000);
+        SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withDateRanges(dateRange).build();
+        SearchResponse res_search = client.search(searchParam);
+        assert (!res_search.getResponse().ok());
+    }
+
+}

tests/milvus-java-test/src/main/java/com/TestTable.java

@@ -0,0 +1,142 @@
+package com;
+
+
+import io.milvus.client.*;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.util.List;
+
+public class TestTable {
+    int index_file_size = 50;
+    int dimension = 128;
+
+    @Test(dataProvider = "ConnectInstance", dataProviderClass = MainClass.class)
+    public void test_create_table(MilvusClient client, String tableName){
+        TableSchema tableSchema = new TableSchema.Builder(tableName, dimension)
+                .withIndexFileSize(index_file_size)
+                .withMetricType(MetricType.L2)
+                .build();
+        Response res = client.createTable(tableSchema);
+        assert(res.ok());
+        Assert.assertEquals(res.ok(), true);
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_create_table_disconnect(MilvusClient client, String tableName){
+        TableSchema tableSchema = new TableSchema.Builder(tableName, dimension)
+                .withIndexFileSize(index_file_size)
+                .withMetricType(MetricType.L2)
+                .build();
+        Response res = client.createTable(tableSchema);
+        assert(!res.ok());
+    }
+
+    @Test(dataProvider = "ConnectInstance", dataProviderClass = MainClass.class)
+    public void test_create_table_repeatably(MilvusClient client, String tableName){
+        TableSchema tableSchema = new TableSchema.Builder(tableName, dimension)
+                .withIndexFileSize(index_file_size)
+                .withMetricType(MetricType.L2)
+                .build();
+        Response res = client.createTable(tableSchema);
+        Assert.assertEquals(res.ok(), true);
+        Response res_new = client.createTable(tableSchema);
+        Assert.assertEquals(res_new.ok(), false);
+    }
+
+    @Test(dataProvider = "ConnectInstance", dataProviderClass = MainClass.class)
+    public void test_create_table_wrong_params(MilvusClient client, String tableName){
+        Integer dimension = 0;
+        TableSchema tableSchema = new TableSchema.Builder(tableName, dimension)
+                .withIndexFileSize(index_file_size)
+                .withMetricType(MetricType.L2)
+                .build();
+        Response res = client.createTable(tableSchema);
+        System.out.println(res.toString());
+        Assert.assertEquals(res.ok(), false);
+    }
+
+    @Test(dataProvider = "ConnectInstance", dataProviderClass = MainClass.class)
+    public void test_show_tables(MilvusClient client, String tableName){
+        Integer tableNum = 10;
+        ShowTablesResponse res = null;
+        for (int i = 0; i < tableNum; ++i) {
+            String tableNameNew = tableName+"_"+Integer.toString(i);
+            TableSchema tableSchema = new TableSchema.Builder(tableNameNew, dimension)
+                    .withIndexFileSize(index_file_size)
+                    .withMetricType(MetricType.L2)
+                    .build();
+            client.createTable(tableSchema);
+            List<String> tableNames = client.showTables().getTableNames();
+            Assert.assertTrue(tableNames.contains(tableNameNew));
+        }
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_show_tables_without_connect(MilvusClient client, String tableName){
+        ShowTablesResponse res = client.showTables();
+        assert(!res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_drop_table(MilvusClient client, String tableName) throws InterruptedException {
+        Response res = client.dropTable(tableName);
+        assert(res.ok());
+        Thread.currentThread().sleep(1000);
+        List<String> tableNames = client.showTables().getTableNames();
+        Assert.assertFalse(tableNames.contains(tableName));
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_drop_table_not_existed(MilvusClient client, String tableName) throws InterruptedException {
+        Response res = client.dropTable(tableName+"_");
+        assert(!res.ok());
+        List<String> tableNames = client.showTables().getTableNames();
+        Assert.assertTrue(tableNames.contains(tableName));
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_drop_table_without_connect(MilvusClient client, String tableName) throws InterruptedException {
+        Response res = client.dropTable(tableName);
+        assert(!res.ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_describe_table(MilvusClient client, String tableName) throws InterruptedException {
+        DescribeTableResponse res = client.describeTable(tableName);
+        assert(res.getResponse().ok());
+        TableSchema tableSchema = res.getTableSchema().get();
+        Assert.assertEquals(tableSchema.getDimension(), dimension);
+        Assert.assertEquals(tableSchema.getTableName(), tableName);
+        Assert.assertEquals(tableSchema.getIndexFileSize(), index_file_size);
+        Assert.assertEquals(tableSchema.getMetricType().name(), tableName.substring(0,2));
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_describe_table_without_connect(MilvusClient client, String tableName) throws InterruptedException {
+        DescribeTableResponse res = client.describeTable(tableName);
+        assert(!res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_has_table_not_existed(MilvusClient client, String tableName) throws InterruptedException {
+        HasTableResponse res = client.hasTable(tableName+"_");
+        assert(res.getResponse().ok());
+        Assert.assertFalse(res.hasTable());
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_has_table_without_connect(MilvusClient client, String tableName) throws InterruptedException {
+        HasTableResponse res = client.hasTable(tableName);
+        assert(!res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_has_table(MilvusClient client, String tableName) throws InterruptedException {
+        HasTableResponse res = client.hasTable(tableName);
+        assert(res.getResponse().ok());
+        Assert.assertTrue(res.hasTable());
+    }
+
+
+}

tests/milvus-java-test/src/main/java/com/TestTableCount.java

@@ -0,0 +1,83 @@
+package com;
+
+import io.milvus.client.*;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+public class TestTableCount {
+    int index_file_size = 50;
+    int dimension = 128;
+
+    public List<List<Float>> gen_vectors(Integer nb) {
+        List<List<Float>> xb = new ArrayList<>();
+        Random random = new Random();
+        for (int i = 0; i < nb; ++i) {
+            ArrayList<Float> vector = new ArrayList<>();
+            for (int j = 0; j < dimension; j++) {
+                vector.add(random.nextFloat());
+            }
+            xb.add(vector);
+        }
+        return xb;
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_table_count_no_vectors(MilvusClient client, String tableName) {
+        Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), 0);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_table_count_table_not_existed(MilvusClient client, String tableName) {
+        GetTableRowCountResponse res = client.getTableRowCount(tableName+"_");
+        assert(!res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class)
+    public void test_table_count_without_connect(MilvusClient client, String tableName) {
+        GetTableRowCountResponse res = client.getTableRowCount(tableName+"_");
+        assert(!res.getResponse().ok());
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_table_count(MilvusClient client, String tableName) throws InterruptedException {
+        int nb = 10000;
+        List<List<Float>> vectors = gen_vectors(nb);
+        // Add vectors
+        InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build();;
+        client.insert(insertParam);
+        Thread.currentThread().sleep(2000);
+        Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb);
+    }
+
+    @Test(dataProvider = "Table", dataProviderClass = MainClass.class)
+    public void test_table_count_multi_tables(MilvusClient client, String tableName) throws InterruptedException {
+        int nb = 10000;
+        List<List<Float>> vectors = gen_vectors(nb);
+        Integer tableNum = 10;
+        GetTableRowCountResponse res = null;
+        for (int i = 0; i < tableNum; ++i) {
+            String tableNameNew = tableName + "_" + Integer.toString(i);
+            TableSchema tableSchema = new TableSchema.Builder(tableNameNew, dimension)
+                    .withIndexFileSize(index_file_size)
+                    .withMetricType(MetricType.L2)
+                    .build();
+            client.createTable(tableSchema);
+            // Add vectors
+            InsertParam insertParam = new InsertParam.Builder(tableNameNew, vectors).build();
+            client.insert(insertParam);
+        }
+        Thread.currentThread().sleep(1000);
+        for (int i = 0; i < tableNum; ++i) {
+            String tableNameNew = tableName + "_" + Integer.toString(i);
+            res = client.getTableRowCount(tableNameNew);
+            Assert.assertEquals(res.getTableRowCount(), nb);
+        }
+    }
+
+}
+
+

tests/milvus-java-test/testng.xml

@@ -0,0 +1,8 @@
+<suite name="Test-class Suite">
+    <test name="Test-class test" >
+        <classes>
+            <class name="com.TestConnect" />
+            <class name="com.TestMix" />
+        </classes>
+    </test>
+</suite>

tests/milvus_ann_acc/.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+logs/

tests/milvus_ann_acc/README.md

@@ -0,0 +1,21 @@
+# Requirements
+
+- python 3.6+
+- pip install -r requirements.txt
+
+# How to use this Test Project
+
+This project is used to test search accuracy based on the given datasets (https://github.com/erikbern/ann-benchmarks#data-sets)
+
+1. start your milvus server
+2. update your test configuration in test.py
+3. run command
+
+```shell
+python test.py
+```
+
+# Contribution getting started
+
+- Follow PEP-8 for naming and black for formatting.
+

tests/milvus_ann_acc/client.py

@@ -0,0 +1,149 @@
+import pdb
+import random
+import logging
+import json
+import time, datetime
+from multiprocessing import Process
+import numpy
+import sklearn.preprocessing
+from milvus import Milvus, IndexType, MetricType
+
+logger = logging.getLogger("milvus_ann_acc.client")
+
+SERVER_HOST_DEFAULT = "127.0.0.1"
+SERVER_PORT_DEFAULT = 19530
+
+
+def time_wrapper(func):
+    """
+    This decorator prints the execution time for the decorated function.
+    """
+    def wrapper(*args, **kwargs):
+        start = time.time()
+        result = func(*args, **kwargs)
+        end = time.time()
+        logger.info("Milvus {} run in {}s".format(func.__name__, round(end - start, 2)))
+        return result
+    return wrapper
+
+
+class MilvusClient(object):
+    def __init__(self, table_name=None, ip=None, port=None):
+        self._milvus = Milvus()
+        self._table_name = table_name
+        try:
+            if not ip:
+                self._milvus.connect(
+                    host = SERVER_HOST_DEFAULT,
+                    port = SERVER_PORT_DEFAULT)
+            else:
+                self._milvus.connect(
+                    host = ip,
+                    port = port)
+        except Exception as e:
+            raise e
+
+    def __str__(self):
+        return 'Milvus table %s' % self._table_name
+
+    def check_status(self, status):
+        if not status.OK():
+            logger.error(status.message)
+            raise Exception("Status not ok")
+
+    def create_table(self, table_name, dimension, index_file_size, metric_type):
+        if not self._table_name:
+            self._table_name = table_name
+        if metric_type == "l2":
+            metric_type = MetricType.L2
+        elif metric_type == "ip":
+            metric_type = MetricType.IP
+        else:
+            logger.error("Not supported metric_type: %s" % metric_type)
+        self._metric_type = metric_type
+        create_param = {'table_name': table_name,
+                 'dimension': dimension,
+                 'index_file_size': index_file_size, 
+                 "metric_type": metric_type}
+        status = self._milvus.create_table(create_param)
+        self.check_status(status)
+
+    @time_wrapper
+    def insert(self, X, ids):
+        if self._metric_type == MetricType.IP:
+            logger.info("Set normalize for metric_type: Inner Product")
+            X = sklearn.preprocessing.normalize(X, axis=1, norm='l2')
+        X = X.astype(numpy.float32)
+        status, result = self._milvus.add_vectors(self._table_name, X.tolist(), ids=ids)
+        self.check_status(status)
+        return status, result
+
+    @time_wrapper
+    def create_index(self, index_type, nlist):
+        if index_type == "flat":
+            index_type = IndexType.FLAT
+        elif index_type == "ivf_flat":
+            index_type = IndexType.IVFLAT
+        elif index_type == "ivf_sq8":
+            index_type = IndexType.IVF_SQ8
+        elif index_type == "ivf_sq8h":
+            index_type = IndexType.IVF_SQ8H
+        elif index_type == "mix_nsg":
+            index_type = IndexType.MIX_NSG
+        index_params = {
+            "index_type": index_type,
+            "nlist": nlist,
+        }
+        logger.info("Building index start, table_name: %s, index_params: %s" % (self._table_name, json.dumps(index_params)))
+        status = self._milvus.create_index(self._table_name, index=index_params, timeout=6*3600)
+        self.check_status(status)
+
+    def describe_index(self):
+        return self._milvus.describe_index(self._table_name)
+
+    def drop_index(self):
+        logger.info("Drop index: %s" % self._table_name)
+        return self._milvus.drop_index(self._table_name)
+
+    @time_wrapper
+    def query(self, X, top_k, nprobe):
+        if self._metric_type == MetricType.IP:
+            logger.info("Set normalize for metric_type: Inner Product")
+            X = sklearn.preprocessing.normalize(X, axis=1, norm='l2')
+        X = X.astype(numpy.float32)
+        status, results = self._milvus.search_vectors(self._table_name, top_k, nprobe, X.tolist())
+        self.check_status(status)
+        # logger.info(results[0])
+        ids = []
+        for result in results:
+            tmp_ids = []
+            for item in result:
+                tmp_ids.append(item.id)
+            ids.append(tmp_ids)
+        return ids
+
+    def count(self):
+        return self._milvus.get_table_row_count(self._table_name)[1]
+
+    def delete(self, timeout=60):
+        logger.info("Start delete table: %s" % self._table_name)
+        self._milvus.delete_table(self._table_name)
+        i = 0
+        while i < timeout:
+            if self.count():
+                time.sleep(1)
+                i = i + 1
+            else:
+                break
+        if i >= timeout:
+            logger.error("Delete table timeout")
+
+    def describe(self):
+        return self._milvus.describe_table(self._table_name)
+
+    def exists_table(self):
+        return self._milvus.has_table(self._table_name)
+
+    @time_wrapper
+    def preload_table(self):
+        return self._milvus.preload_table(self._table_name)

tests/milvus_ann_acc/config.yaml

@@ -0,0 +1,17 @@
+datasets: 
+  sift-128-euclidean: 
+    cpu_cache_size: 16
+    gpu_cache_size: 5
+    index_file_size: [1024]
+  nytimes-16-angular:
+    cpu_cache_size: 16
+    gpu_cache_size: 5
+    index_file_size: [1024]
+
+index:
+  index_types: ['flat', 'ivf_flat', 'ivf_sq8']
+  nlists: [8092, 16384]
+  
+search:
+  nprobes: [1, 8, 32]
+  top_ks: [10]

tests/milvus_ann_acc/main.py

@@ -0,0 +1,26 @@
+
+import argparse
+
+def main():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument(
+        '--dataset',
+        metavar='NAME',
+        help='the dataset to load training points from',
+        default='glove-100-angular',
+        choices=DATASETS.keys())
+    parser.add_argument(
+        "-k", "--count",
+        default=10,
+        type=positive_int,
+        help="the number of near neighbours to search for")
+    parser.add_argument(
+        '--definitions',
+        metavar='FILE',
+        help='load algorithm definitions from FILE',
+        default='algos.yaml')
+    parser.add_argument(
+        '--image-tag',
+        default=None,
+        help='pull image first')

tests/milvus_ann_acc/requirements.txt

@@ -0,0 +1,4 @@
+numpy==1.16.3
+pymilvus>=0.2.0
+scikit-learn==0.19.1
+h5py==2.7.1

tests/milvus_ann_acc/test.py

@@ -0,0 +1,132 @@
+import os
+import pdb
+import time
+import random
+import sys
+import h5py
+import numpy
+import logging
+from logging import handlers
+
+from client import MilvusClient
+
+LOG_FOLDER = "logs"
+logger = logging.getLogger("milvus_ann_acc")
+
+formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s')
+if not os.path.exists(LOG_FOLDER):
+    os.system('mkdir -p %s' % LOG_FOLDER)
+fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'acc'), "D", 1, 10)
+fileTimeHandler.suffix = "%Y%m%d.log"
+fileTimeHandler.setFormatter(formatter)
+logging.basicConfig(level=logging.DEBUG)
+fileTimeHandler.setFormatter(formatter)
+logger.addHandler(fileTimeHandler)
+
+
+def get_dataset_fn(dataset_name):
+    file_path = "/test/milvus/ann_hdf5/"
+    if not os.path.exists(file_path):
+        raise Exception("%s not exists" % file_path)
+    return os.path.join(file_path, '%s.hdf5' % dataset_name)
+
+
+def get_dataset(dataset_name):
+    hdf5_fn = get_dataset_fn(dataset_name)
+    hdf5_f = h5py.File(hdf5_fn)
+    return hdf5_f
+
+
+def parse_dataset_name(dataset_name):
+    data_type = dataset_name.split("-")[0]
+    dimension = int(dataset_name.split("-")[1])
+    metric = dataset_name.split("-")[-1]
+    # metric = dataset.attrs['distance']
+    # dimension = len(dataset["train"][0])
+    if metric == "euclidean":
+        metric_type = "l2"
+    elif metric  == "angular":
+        metric_type = "ip"
+    return ("ann"+data_type, dimension, metric_type)
+
+
+def get_table_name(dataset_name, index_file_size):
+    data_type, dimension, metric_type = parse_dataset_name(dataset_name)
+    dataset = get_dataset(dataset_name)
+    table_size = len(dataset["train"])
+    table_size = str(table_size // 1000000)+"m"
+    table_name = data_type+'_'+table_size+'_'+str(index_file_size)+'_'+str(dimension)+'_'+metric_type
+    return table_name
+
+
+def main(dataset_name, index_file_size, nlist=16384, force=False):
+    top_k = 10
+    nprobes = [32, 128]
+
+    dataset = get_dataset(dataset_name)
+    table_name = get_table_name(dataset_name, index_file_size)
+    m = MilvusClient(table_name)
+    if m.exists_table():
+        if force is True:
+            logger.info("Re-create table: %s" % table_name)
+            m.delete()
+            time.sleep(10)
+        else:
+            logger.info("Table name: %s existed" % table_name)
+            return
+    data_type, dimension, metric_type = parse_dataset_name(dataset_name)
+    m.create_table(table_name, dimension, index_file_size, metric_type)
+    print(m.describe())
+    vectors = numpy.array(dataset["train"])
+    query_vectors = numpy.array(dataset["test"])
+    # m.insert(vectors)
+
+    interval = 100000
+    loops = len(vectors) // interval + 1
+
+    for i in range(loops):
+        start = i*interval
+        end = min((i+1)*interval, len(vectors))
+        tmp_vectors = vectors[start:end]
+        if start < end:
+            m.insert(tmp_vectors, ids=[i for i in range(start, end)])
+    
+    time.sleep(60)
+    print(m.count())
+
+    for index_type in ["ivf_flat", "ivf_sq8", "ivf_sq8h"]:
+        m.create_index(index_type, nlist)
+        print(m.describe_index())
+        if m.count() != len(vectors):
+            return
+        m.preload_table()
+        true_ids = numpy.array(dataset["neighbors"])
+        for nprobe in nprobes:
+            print("nprobe: %s" % nprobe)
+            sum_radio = 0.0; avg_radio = 0.0
+            result_ids = m.query(query_vectors, top_k, nprobe)
+            # print(result_ids[:10])
+            for index, result_item in enumerate(result_ids):
+                if len(set(true_ids[index][:top_k])) != len(set(result_item)):
+                    logger.info("Error happened")
+                    # logger.info(query_vectors[index])
+                    # logger.info(true_ids[index][:top_k], result_item)
+                tmp = set(true_ids[index][:top_k]).intersection(set(result_item))
+                sum_radio = sum_radio + (len(tmp) / top_k)
+            avg_radio = round(sum_radio / len(result_ids), 4) 
+            logger.info(avg_radio)
+        m.drop_index()
+
+
+if __name__ == "__main__":
+    print("glove-25-angular")
+    # main("sift-128-euclidean", 1024, force=True)
+    for index_file_size in [50, 1024]:
+        print("Index file size: %d" % index_file_size)
+        main("glove-25-angular", index_file_size, force=True)
+
+    print("sift-128-euclidean")
+    for index_file_size in [50, 1024]:
+        print("Index file size: %d" % index_file_size)
+        main("sift-128-euclidean", index_file_size, force=True)
+    # m = MilvusClient()

tests/milvus_benchmark/.gitignore

@@ -0,0 +1,8 @@
+random_data
+benchmark_logs/
+db/
+logs/
+*idmap*.txt
+__pycache__/
+venv
+.idea

tests/milvus_benchmark/README.md

@@ -0,0 +1,23 @@
+# Requirements
+
+- python 3.6+
+- pip install -r requirements.txt
+
+# How to use this Test Project
+
+This project is used to test performance / accuracy /  stability of milvus server
+
+1. update your test configuration in suites_*.yaml
+2. run command
+
+```shell
+### docker mode:
+python main.py --image=milvusdb/milvus:latest --run-count=2 --run-type=performance
+
+### local mode:
+python main.py --local --run-count=2 --run-type=performance --ip=127.0.0.1 --port=19530
+```
+
+# Contribution getting started
+
+- Follow PEP-8 for naming and black for formatting.

tests/milvus_benchmark/client.py

@@ -0,0 +1,244 @@
+import pdb
+import random
+import logging
+import json
+import sys
+import time, datetime
+from multiprocessing import Process
+from milvus import Milvus, IndexType, MetricType
+
+logger = logging.getLogger("milvus_benchmark.client")
+
+SERVER_HOST_DEFAULT = "127.0.0.1"
+SERVER_PORT_DEFAULT = 19530
+
+
+def time_wrapper(func):
+    """
+    This decorator prints the execution time for the decorated function.
+    """
+    def wrapper(*args, **kwargs):
+        start = time.time()
+        result = func(*args, **kwargs)
+        end = time.time()
+        logger.info("Milvus {} run in {}s".format(func.__name__, round(end - start, 2)))
+        return result
+    return wrapper
+
+
+class MilvusClient(object):
+    def __init__(self, table_name=None, ip=None, port=None):
+        self._milvus = Milvus()
+        self._table_name = table_name
+        try:
+            if not ip:
+                self._milvus.connect(
+                    host = SERVER_HOST_DEFAULT,
+                    port = SERVER_PORT_DEFAULT)
+            else:
+                self._milvus.connect(
+                    host = ip,
+                    port = port)
+        except Exception as e:
+            raise e
+
+    def __str__(self):
+        return 'Milvus table %s' % self._table_name
+
+    def check_status(self, status):
+        if not status.OK():
+            logger.error(status.message)
+            raise Exception("Status not ok")
+
+    def create_table(self, table_name, dimension, index_file_size, metric_type):
+        if not self._table_name:
+            self._table_name = table_name
+        if metric_type == "l2":
+            metric_type = MetricType.L2
+        elif metric_type == "ip":
+            metric_type = MetricType.IP
+        else:
+            logger.error("Not supported metric_type: %s" % metric_type)
+        create_param = {'table_name': table_name,
+                 'dimension': dimension,
+                 'index_file_size': index_file_size, 
+                 "metric_type": metric_type}
+        status = self._milvus.create_table(create_param)
+        self.check_status(status)
+
+    @time_wrapper
+    def insert(self, X, ids=None):
+        status, result = self._milvus.add_vectors(self._table_name, X, ids)
+        self.check_status(status)
+        return status, result
+
+    @time_wrapper
+    def create_index(self, index_type, nlist):
+        if index_type == "flat":
+            index_type = IndexType.FLAT
+        elif index_type == "ivf_flat":
+            index_type = IndexType.IVFLAT
+        elif index_type == "ivf_sq8":
+            index_type = IndexType.IVF_SQ8
+        elif index_type == "mix_nsg":
+            index_type = IndexType.MIX_NSG
+        elif index_type == "ivf_sq8h":
+            index_type = IndexType.IVF_SQ8H
+        index_params = {
+            "index_type": index_type,
+            "nlist": nlist,
+        }
+        logger.info("Building index start, table_name: %s, index_params: %s" % (self._table_name, json.dumps(index_params)))
+        status = self._milvus.create_index(self._table_name, index=index_params, timeout=6*3600)
+        self.check_status(status)
+
+    def describe_index(self):
+        return self._milvus.describe_index(self._table_name)
+
+    def drop_index(self):
+        logger.info("Drop index: %s" % self._table_name)
+        return self._milvus.drop_index(self._table_name)
+
+    @time_wrapper
+    def query(self, X, top_k, nprobe):
+        status, result = self._milvus.search_vectors(self._table_name, top_k, nprobe, X)
+        self.check_status(status)
+        return status, result
+
+    def count(self):
+        return self._milvus.get_table_row_count(self._table_name)[1]
+
+    def delete(self, timeout=60):
+        logger.info("Start delete table: %s" % self._table_name)
+        self._milvus.delete_table(self._table_name)
+        i = 0
+        while i < timeout:
+            if self.count():
+                time.sleep(1)
+                i = i + 1
+                continue
+            else:
+                break
+        if i < timeout:
+            logger.error("Delete table timeout")
+
+    def describe(self):
+        return self._milvus.describe_table(self._table_name)
+
+    def exists_table(self):
+        return self._milvus.has_table(self._table_name)
+
+    @time_wrapper
+    def preload_table(self):
+        return self._milvus.preload_table(self._table_name, timeout=3000)
+
+
+def fit(table_name, X):
+    milvus = Milvus()
+    milvus.connect(host = SERVER_HOST_DEFAULT, port = SERVER_PORT_DEFAULT) 
+    start = time.time()
+    status, ids = milvus.add_vectors(table_name, X)
+    end = time.time()
+    logger(status, round(end - start, 2))
+
+
+def fit_concurrent(table_name, process_num, vectors):
+    processes = []
+
+    for i in range(process_num):
+        p = Process(target=fit, args=(table_name, vectors, ))
+        processes.append(p)
+        p.start()
+    for p in processes:             
+        p.join()
+    
+
+if __name__ == "__main__":
+
+    # table_name = "sift_2m_20_128_l2"
+    table_name = "test_tset1"
+    m = MilvusClient(table_name)
+    # m.create_table(table_name, 128, 50, "l2")
+
+    print(m.describe())
+    # print(m.count())
+    # print(m.describe_index())
+    insert_vectors = [[random.random() for _ in range(128)] for _ in range(10000)]
+    for i in range(5):
+        m.insert(insert_vectors)
+    print(m.create_index("ivf_sq8h", 16384))
+    X = [insert_vectors[0]]
+    top_k = 10
+    nprobe = 10
+    print(m.query(X, top_k, nprobe))
+
+    # # # print(m.drop_index())
+    # # print(m.describe_index())
+    # # sys.exit()
+    # # # insert_vectors = [[random.random() for _ in range(128)] for _ in range(100000)]
+    # # # for i in range(100):
+    # # #     m.insert(insert_vectors)
+    # # # time.sleep(5)
+    # # # print(m.describe_index())
+    # # # print(m.drop_index())
+    # # m.create_index("ivf_sq8h", 16384)
+    # print(m.count())
+    # print(m.describe_index())
+
+
+
+    # sys.exit()
+    # print(m.create_index("ivf_sq8h", 16384))
+    # print(m.count())
+    # print(m.describe_index())
+    import numpy as np
+
+    def mmap_fvecs(fname):
+        x = np.memmap(fname, dtype='int32', mode='r')
+        d = x[0]
+        return x.view('float32').reshape(-1, d + 1)[:, 1:]
+
+    print(mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs"))
+    # SIFT_SRC_QUERY_DATA_DIR = '/poc/yuncong/ann_1000m'
+    # file_name = SIFT_SRC_QUERY_DATA_DIR+'/'+'query.npy'
+    # data = numpy.load(file_name)
+    # query_vectors = data[0:2].tolist()
+    # print(len(query_vectors))
+    # results = m.query(query_vectors, 10, 10)
+    # result_ids = []
+    # for result in results[1]:
+    #     tmp = []
+    #     for item in result:
+    #         tmp.append(item.id)
+    #     result_ids.append(tmp)
+    # print(result_ids[0][:10])
+    # # gt
+    # file_name = SIFT_SRC_QUERY_DATA_DIR+"/gnd/"+"idx_1M.ivecs"
+    # a = numpy.fromfile(file_name, dtype='int32')
+    # d = a[0]
+    # true_ids = a.reshape(-1, d + 1)[:, 1:].copy()
+    # print(true_ids[:3, :2])
+
+    # print(len(true_ids[0]))
+    # import numpy as np
+    # import sklearn.preprocessing
+
+    # def mmap_fvecs(fname):
+    #     x = np.memmap(fname, dtype='int32', mode='r')
+    #     d = x[0]
+    #     return x.view('float32').reshape(-1, d + 1)[:, 1:]
+
+    # data = mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs")
+    # print(data[0], len(data[0]), len(data))
+
+    # total_size = 10000
+    # # total_size = 1000000000
+    # file_size = 1000
+    # # file_size = 100000
+    # file_num = total_size // file_size
+    # for i in range(file_num):
+    #     fname = "/test/milvus/raw_data/deep1b/binary_96_%05d" % i
+    #     print(fname, i*file_size, (i+1)*file_size)
+    #     single_data = data[i*file_size : (i+1)*file_size]
+    #     single_data = sklearn.preprocessing.normalize(single_data, axis=1, norm='l2')
+    #     np.save(fname, single_data)

tests/milvus_benchmark/conf/log_config.conf

@@ -0,0 +1,28 @@
+* GLOBAL:
+    FORMAT                  =   "%datetime | %level | %logger | %msg"
+    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-global.log"
+    ENABLED                 =   true
+    TO_FILE                 =   true
+    TO_STANDARD_OUTPUT      =   false
+    SUBSECOND_PRECISION     =   3
+    PERFORMANCE_TRACKING    =   false
+    MAX_LOG_FILE_SIZE       =   2097152 ## Throw log files away after 2MB
+* DEBUG:
+    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-debug.log"
+    ENABLED                 =   true
+* WARNING:
+    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-warning.log"
+* TRACE:
+    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-trace.log"
+* VERBOSE:
+    FORMAT                  =   "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
+    TO_FILE                 =   false
+    TO_STANDARD_OUTPUT      =   false
+## Error logs
+* ERROR:
+    ENABLED                 =   true
+    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-error.log"
+* FATAL:
+    ENABLED                 =   true
+    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-fatal.log"
+

tests/milvus_benchmark/conf/server_config.yaml

@@ -0,0 +1,28 @@
+cache_config:
+  cache_insert_data: false
+  cpu_cache_capacity: 16
+  gpu_cache_capacity: 6
+  cpu_cache_threshold: 0.85
+db_config:
+  backend_url: sqlite://:@:/
+  build_index_gpu: 0
+  insert_buffer_size: 4
+  preload_table: null
+  primary_path: /opt/milvus
+  secondary_path: null
+engine_config:
+  use_blas_threshold: 20
+metric_config:
+  collector: prometheus
+  enable_monitor: true
+  prometheus_config:
+    port: 8080
+resource_config:
+  resource_pool:
+  - cpu
+  - gpu0
+server_config:
+  address: 0.0.0.0
+  deploy_mode: single
+  port: 19530
+  time_zone: UTC+8

tests/milvus_benchmark/conf/server_config.yaml.cpu

@@ -0,0 +1,31 @@
+server_config:
+  address: 0.0.0.0            
+  port: 19530                 
+  deploy_mode: single                
+  time_zone: UTC+8  
+            
+db_config:
+  primary_path: /opt/milvus             
+  secondary_path:                        
+  backend_url: sqlite://:@:/    
+  insert_buffer_size: 4               
+  build_index_gpu: 0  
+  preload_table:             
+
+metric_config:
+  enable_monitor: false             
+  collector: prometheus             
+  prometheus_config:
+    port: 8080        
+
+cache_config:
+  cpu_cache_capacity: 16           
+  cpu_cache_threshold: 0.85     
+  cache_insert_data: false 
+
+engine_config:
+  use_blas_threshold: 20            
+
+resource_config:
+  resource_pool:
+    - cpu

tests/milvus_benchmark/conf/server_config.yaml.multi

@@ -0,0 +1,33 @@
+server_config:
+  address: 0.0.0.0            
+  port: 19530                 
+  deploy_mode: single                
+  time_zone: UTC+8  
+            
+db_config:
+  primary_path: /opt/milvus             
+  secondary_path:                        
+  backend_url: sqlite://:@:/    
+  insert_buffer_size: 4               
+  build_index_gpu: 0  
+  preload_table:             
+
+metric_config:
+  enable_monitor: false             
+  collector: prometheus             
+  prometheus_config:
+    port: 8080        
+
+cache_config:
+  cpu_cache_capacity: 16           
+  cpu_cache_threshold: 0.85     
+  cache_insert_data: false 
+
+engine_config:
+  use_blas_threshold: 20            
+
+resource_config:
+  resource_pool:
+    - cpu
+    - gpu0
+    - gpu1

tests/milvus_benchmark/conf/server_config.yaml.single

@@ -0,0 +1,32 @@
+server_config:
+  address: 0.0.0.0            
+  port: 19530                 
+  deploy_mode: single                
+  time_zone: UTC+8  
+            
+db_config:
+  primary_path: /opt/milvus             
+  secondary_path:                        
+  backend_url: sqlite://:@:/    
+  insert_buffer_size: 4               
+  build_index_gpu: 0  
+  preload_table:       
+
+metric_config:
+  enable_monitor: false             
+  collector: prometheus             
+  prometheus_config:
+    port: 8080        
+
+cache_config:
+  cpu_cache_capacity: 16           
+  cpu_cache_threshold: 0.85     
+  cache_insert_data: false 
+
+engine_config:
+  use_blas_threshold: 20            
+
+resource_config:
+  resource_pool:
+    - cpu
+    - gpu0

tests/milvus_benchmark/demo.py

@@ -0,0 +1,51 @@
+import os
+import logging
+import pdb
+import time
+import random
+from multiprocessing import Process
+import numpy as np
+from client import MilvusClient
+
+nq = 100000
+dimension = 128
+run_count = 1
+table_name = "sift_10m_1024_128_ip"
+insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)]
+
+def do_query(milvus, table_name, top_ks, nqs, nprobe, run_count):
+    bi_res = []
+    for index, nq in enumerate(nqs):
+        tmp_res = []
+        for top_k in top_ks:
+            avg_query_time = 0.0
+            total_query_time = 0.0
+            vectors = insert_vectors[0:nq]
+            for i in range(run_count):
+                start_time = time.time()
+                status, query_res = milvus.query(vectors, top_k, nprobe)
+                total_query_time = total_query_time + (time.time() - start_time)
+                if status.code:
+                    print(status.message)
+            avg_query_time = round(total_query_time / run_count, 2)
+            tmp_res.append(avg_query_time)
+        bi_res.append(tmp_res)
+    return bi_res
+
+while 1:
+    milvus_instance = MilvusClient(table_name, ip="192.168.1.197", port=19530)
+    top_ks = random.sample([x for x in range(1, 100)], 4)
+    nqs = random.sample([x for x in range(1, 1000)], 3)
+    nprobe = random.choice([x for x in range(1, 500)])
+    res = do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count)
+    status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))])
+    if not status.OK():
+        logger.error(status.message)
+
+    # status = milvus_instance.drop_index()
+    if not status.OK():
+        print(status.message)
+    index_type = "ivf_sq8"
+    status = milvus_instance.create_index(index_type, 16384)
+    if not status.OK():
+        print(status.message)

tests/milvus_benchmark/docker_runner.py

@@ -0,0 +1,261 @@
+import os
+import logging
+import pdb
+import time
+import random
+from multiprocessing import Process
+import numpy as np
+from client import MilvusClient
+import utils
+import parser
+from runner import Runner
+
+logger = logging.getLogger("milvus_benchmark.docker")
+
+
+class DockerRunner(Runner):
+    """run docker mode"""
+    def __init__(self, image):
+        super(DockerRunner, self).__init__()
+        self.image = image
+        
+    def run(self, definition, run_type=None):
+        if run_type == "performance":
+            for op_type, op_value in definition.items():
+                # run docker mode
+                run_count = op_value["run_count"]
+                run_params = op_value["params"]
+                container = None
+                
+                if op_type == "insert":
+                    for index, param in enumerate(run_params):
+                        logger.info("Definition param: %s" % str(param))
+                        table_name = param["table_name"]
+                        volume_name = param["db_path_prefix"]
+                        print(table_name)
+                        (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+                        for k, v in param.items():
+                            if k.startswith("server."):
+                                # Update server config
+                                utils.modify_config(k, v, type="server", db_slave=None)
+                        container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
+                        time.sleep(2)
+                        milvus = MilvusClient(table_name)
+                        # Check has table or not
+                        if milvus.exists_table():
+                            milvus.delete()
+                            time.sleep(10)
+                        milvus.create_table(table_name, dimension, index_file_size, metric_type)
+                        res = self.do_insert(milvus, table_name, data_type, dimension, table_size, param["ni_per"])
+                        logger.info(res)
+
+                        # wait for file merge
+                        time.sleep(6 * (table_size / 500000))
+                        # Clear up
+                        utils.remove_container(container)
+
+                elif op_type == "query":
+                    for index, param in enumerate(run_params):
+                        logger.info("Definition param: %s" % str(param))
+                        table_name = param["dataset"]
+                        volume_name = param["db_path_prefix"]
+                        (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+                        for k, v in param.items():
+                            if k.startswith("server."):                   
+                                utils.modify_config(k, v, type="server")
+                        container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
+                        time.sleep(2)
+                        milvus = MilvusClient(table_name)
+                        logger.debug(milvus._milvus.show_tables())
+                        # Check has table or not
+                        if not milvus.exists_table():
+                            logger.warning("Table %s not existed, continue exec next params ..." % table_name)
+                            continue
+                        # parse index info
+                        index_types = param["index.index_types"]
+                        nlists = param["index.nlists"]
+                        # parse top-k, nq, nprobe
+                        top_ks, nqs, nprobes = parser.search_params_parser(param)
+                        for index_type in index_types:
+                            for nlist in nlists:
+                                result = milvus.describe_index()
+                                logger.info(result)
+                                milvus.create_index(index_type, nlist)
+                                result = milvus.describe_index()
+                                logger.info(result)
+                                # preload index
+                                milvus.preload_table()
+                                logger.info("Start warm up query")
+                                res = self.do_query(milvus, table_name, [1], [1], 1, 1)
+                                logger.info("End warm up query")
+                                # Run query test
+                                for nprobe in nprobes:
+                                    logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
+                                    res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count)
+                                    headers = ["Nprobe/Top-k"]
+                                    headers.extend([str(top_k) for top_k in top_ks])
+                                    utils.print_table(headers, nqs, res)
+                        utils.remove_container(container)
+
+        elif run_type == "accuracy":
+            """
+            {
+                "dataset": "random_50m_1024_512", 
+                "index.index_types": ["flat", ivf_flat", "ivf_sq8"],
+                "index.nlists": [16384],
+                "nprobes": [1, 32, 128], 
+                "nqs": [100],
+                "top_ks": [1, 64], 
+                "server.use_blas_threshold": 1100, 
+                "server.cpu_cache_capacity": 256
+            }
+            """
+            for op_type, op_value in definition.items():
+                if op_type != "query":
+                    logger.warning("invalid operation: %s in accuracy test, only support query operation" % op_type)
+                    break
+                run_count = op_value["run_count"]
+                run_params = op_value["params"]
+                container = None
+
+                for index, param in enumerate(run_params):
+                    logger.info("Definition param: %s" % str(param))
+                    table_name = param["dataset"]
+                    sift_acc = False
+                    if "sift_acc" in param:
+                        sift_acc = param["sift_acc"]
+                    (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+                    for k, v in param.items():
+                        if k.startswith("server."):                   
+                            utils.modify_config(k, v, type="server")
+                    volume_name = param["db_path_prefix"]
+                    container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
+                    time.sleep(2)
+                    milvus = MilvusClient(table_name)
+                    # Check has table or not
+                    if not milvus.exists_table():
+                        logger.warning("Table %s not existed, continue exec next params ..." % table_name)
+                        continue
+
+                    # parse index info
+                    index_types = param["index.index_types"]
+                    nlists = param["index.nlists"]
+                    # parse top-k, nq, nprobe
+                    top_ks, nqs, nprobes = parser.search_params_parser(param)
+
+                    if sift_acc is True:
+                        # preload groundtruth data
+                        true_ids_all = self.get_groundtruth_ids(table_size)
+
+                    acc_dict = {}
+                    for index_type in index_types:
+                        for nlist in nlists:
+                            result = milvus.describe_index()
+                            logger.info(result)
+                            milvus.create_index(index_type, nlist)
+                            # preload index
+                            milvus.preload_table()
+                            # Run query test
+                            for nprobe in nprobes:
+                                logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
+                                for top_k in top_ks:
+                                    for nq in nqs:
+                                        result_ids = []
+                                        id_prefix = "%s_index_%s_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \
+                                                    (table_name, index_type, nlist, metric_type, nprobe, top_k, nq)
+                                        if sift_acc is False:
+                                            self.do_query_acc(milvus, table_name, top_k, nq, nprobe, id_prefix)
+                                            if index_type != "flat":
+                                                # Compute accuracy
+                                                base_name = "%s_index_flat_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \
+                                                    (table_name, nlist, metric_type, nprobe, top_k, nq)
+                                                avg_acc = self.compute_accuracy(base_name, id_prefix)
+                                                logger.info("Query: <%s> accuracy: %s" % (id_prefix, avg_acc))
+                                        else:
+                                            result_ids = self.do_query_ids(milvus, table_name, top_k, nq, nprobe)
+                                            acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids)
+                                            logger.info("Query: <%s> accuracy: %s" % (id_prefix, acc_value))
+                    # # print accuracy table
+                    # headers = [table_name]
+                    # headers.extend([str(top_k) for top_k in top_ks])
+                    # utils.print_table(headers, nqs, res)
+
+                    # remove container, and run next definition
+                    logger.info("remove container, and run next definition")
+                    utils.remove_container(container)
+
+        elif run_type == "stability":
+            for op_type, op_value in definition.items():
+                if op_type != "query":
+                    logger.warning("invalid operation: %s in accuracy test, only support query operation" % op_type)
+                    break
+                run_count = op_value["run_count"]
+                run_params = op_value["params"]
+                container = None
+                for index, param in enumerate(run_params):
+                    logger.info("Definition param: %s" % str(param))
+                    table_name = param["dataset"]
+                    volume_name = param["db_path_prefix"]
+                    (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+                    
+                    # set default test time
+                    if "during_time" not in param:
+                        during_time = 100 # seconds
+                    else:
+                        during_time = int(param["during_time"]) * 60
+                    # set default query process num
+                    if "query_process_num" not in param:
+                        query_process_num = 10
+                    else:
+                        query_process_num = int(param["query_process_num"])
+
+                    for k, v in param.items():
+                        if k.startswith("server."):                   
+                            utils.modify_config(k, v, type="server")
+
+                    container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
+                    time.sleep(2)
+                    milvus = MilvusClient(table_name)
+                    # Check has table or not
+                    if not milvus.exists_table():
+                        logger.warning("Table %s not existed, continue exec next params ..." % table_name)
+                        continue
+
+                    start_time = time.time()
+                    insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(10000)]
+                    while time.time() < start_time + during_time:
+                        processes = []
+                        # do query
+                        # for i in range(query_process_num):
+                        #     milvus_instance = MilvusClient(table_name)
+                        #     top_k = random.choice([x for x in range(1, 100)])
+                        #     nq = random.choice([x for x in range(1, 100)])
+                        #     nprobe = random.choice([x for x in range(1, 1000)])
+                        #     # logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
+                        #     p = Process(target=self.do_query, args=(milvus_instance, table_name, [top_k], [nq], [nprobe], run_count, ))
+                        #     processes.append(p)
+                        #     p.start()
+                        #     time.sleep(0.1)
+                        # for p in processes:
+                        #     p.join()
+                        milvus_instance = MilvusClient(table_name)
+                        top_ks = random.sample([x for x in range(1, 100)], 3)
+                        nqs = random.sample([x for x in range(1, 1000)], 3)
+                        nprobe = random.choice([x for x in range(1, 500)])
+                        res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count)
+                        if int(time.time() - start_time) % 120 == 0:
+                            status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))])
+                            if not status.OK():
+                                logger.error(status)
+                            # status = milvus_instance.drop_index()
+                            # if not status.OK():
+                            #     logger.error(status)
+                            # index_type = random.choice(["flat", "ivf_flat", "ivf_sq8"])
+                            result = milvus.describe_index()
+                            logger.info(result)
+                            milvus_instance.create_index("ivf_sq8", 16384)
+                    utils.remove_container(container)
+
+        else:
+            logger.warning("Run type: %s not supported" % run_type)
+

tests/milvus_benchmark/local_runner.py

@@ -0,0 +1,132 @@
+import os
+import logging
+import pdb
+import time
+import random
+from multiprocessing import Process
+import numpy as np
+from client import MilvusClient
+import utils
+import parser
+from runner import Runner
+
+logger = logging.getLogger("milvus_benchmark.local_runner")
+
+
+class LocalRunner(Runner):
+    """run local mode"""
+    def __init__(self, ip, port):
+        super(LocalRunner, self).__init__()
+        self.ip = ip
+        self.port = port
+
+    def run(self, definition, run_type=None):
+        if run_type == "performance":
+            for op_type, op_value in definition.items():
+                run_count = op_value["run_count"]
+                run_params = op_value["params"]
+
+                if op_type == "insert":
+                    for index, param in enumerate(run_params):
+                        table_name = param["table_name"]
+                        # random_1m_100_512
+                        (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+                        milvus = MilvusClient(table_name, ip=self.ip, port=self.port)
+                        # Check has table or not
+                        if milvus.exists_table():
+                            milvus.delete()
+                            time.sleep(10)
+                        milvus.create_table(table_name, dimension, index_file_size, metric_type)
+                        res = self.do_insert(milvus, table_name, data_type, dimension, table_size, param["ni_per"])
+                        logger.info(res)
+
+                elif op_type == "query":
+                    for index, param in enumerate(run_params):
+                        logger.info("Definition param: %s" % str(param))
+                        table_name = param["dataset"]
+                        (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+
+                        milvus = MilvusClient(table_name, ip=self.ip, port=self.port)
+                        # parse index info
+                        index_types = param["index.index_types"]
+                        nlists = param["index.nlists"]
+                        # parse top-k, nq, nprobe
+                        top_ks, nqs, nprobes = parser.search_params_parser(param)
+
+                        for index_type in index_types:
+                            for nlist in nlists:
+                                milvus.create_index(index_type, nlist)
+                                # preload index
+                                milvus.preload_table()
+                                # Run query test
+                                for nprobe in nprobes:
+                                    logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
+                                    res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count)
+                                    headers = [param["dataset"]]
+                                    headers.extend([str(top_k) for top_k in top_ks])
+                                    utils.print_table(headers, nqs, res)
+
+        elif run_type == "stability":
+            for op_type, op_value in definition.items():
+                if op_type != "query":
+                    logger.warning("invalid operation: %s in accuracy test, only support query operation" % op_type)
+                    break
+                run_count = op_value["run_count"]
+                run_params = op_value["params"]
+                nq = 10000
+
+                for index, param in enumerate(run_params):
+                    logger.info("Definition param: %s" % str(param))
+                    table_name = param["dataset"]
+                    (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+                    
+                    # set default test time
+                    if "during_time" not in param:
+                        during_time = 100 # seconds
+                    else:
+                        during_time = int(param["during_time"]) * 60
+                    # set default query process num
+                    if "query_process_num" not in param:
+                        query_process_num = 10
+                    else:
+                        query_process_num = int(param["query_process_num"])
+                    milvus = MilvusClient(table_name)
+                    # Check has table or not
+                    if not milvus.exists_table():
+                        logger.warning("Table %s not existed, continue exec next params ..." % table_name)
+                        continue
+
+                    start_time = time.time()
+                    insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)]
+                    while time.time() < start_time + during_time:
+                        processes = []
+                        # # do query
+                        # for i in range(query_process_num):
+                        #     milvus_instance = MilvusClient(table_name)
+                        #     top_k = random.choice([x for x in range(1, 100)])
+                        #     nq = random.choice([x for x in range(1, 1000)])
+                        #     nprobe = random.choice([x for x in range(1, 500)])
+                        #     logger.info(nprobe)
+                        #     p = Process(target=self.do_query, args=(milvus_instance, table_name, [top_k], [nq], 64, run_count, ))
+                        #     processes.append(p)
+                        #     p.start()
+                        #     time.sleep(0.1)
+                        # for p in processes:
+                        #     p.join()
+                        milvus_instance = MilvusClient(table_name)
+                        top_ks = random.sample([x for x in range(1, 100)], 4)
+                        nqs = random.sample([x for x in range(1, 1000)], 3)
+                        nprobe = random.choice([x for x in range(1, 500)])
+                        res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count)
+                        # milvus_instance = MilvusClient(table_name)
+                        status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))])
+                        if not status.OK():
+                            logger.error(status.message)
+                        if (time.time() - start_time) % 300 == 0:
+                            status = milvus_instance.drop_index()
+                            if not status.OK():
+                                logger.error(status.message)
+                            index_type = random.choice(["flat", "ivf_flat", "ivf_sq8"])
+                            status = milvus_instance.create_index(index_type, 16384)
+                            if not status.OK():
+                                logger.error(status.message)

tests/milvus_benchmark/main.py

@@ -0,0 +1,131 @@
+import os
+import sys
+import time
+import pdb
+import argparse
+import logging
+import utils
+from yaml import load, dump
+from logging import handlers
+from parser import operations_parser
+from local_runner import LocalRunner
+from docker_runner import DockerRunner
+
+DEFAULT_IMAGE = "milvusdb/milvus:latest"
+LOG_FOLDER = "benchmark_logs"
+logger = logging.getLogger("milvus_benchmark")
+
+formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s')
+if not os.path.exists(LOG_FOLDER):
+    os.system('mkdir -p %s' % LOG_FOLDER)
+fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'milvus_benchmark'), "D", 1, 10)
+fileTimeHandler.suffix = "%Y%m%d.log"
+fileTimeHandler.setFormatter(formatter)
+logging.basicConfig(level=logging.DEBUG)
+fileTimeHandler.setFormatter(formatter)
+logger.addHandler(fileTimeHandler)
+
+
+def positive_int(s):
+    i = None
+    try:
+        i = int(s)
+    except ValueError:
+        pass
+    if not i or i < 1:
+        raise argparse.ArgumentTypeError("%r is not a positive integer" % s)
+    return i
+
+
+# # link random_data if not exists
+# def init_env():
+#     if not os.path.islink(BINARY_DATA_FOLDER):
+#         try:
+#             os.symlink(SRC_BINARY_DATA_FOLDER, BINARY_DATA_FOLDER)
+#         except Exception as e:
+#             logger.error("Create link failed: %s" % str(e))
+#             sys.exit()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument(
+        '--image',
+        help='use the given image')
+    parser.add_argument(
+        '--local',
+        action='store_true',
+        help='use local milvus server')
+    parser.add_argument(
+        "--run-count",
+        default=1,
+        type=positive_int,
+        help="run each db operation times")
+    # performance / stability / accuracy test
+    parser.add_argument(
+        "--run-type",
+        default="performance",
+        help="run type, default performance")
+    parser.add_argument(
+        '--suites',
+        metavar='FILE',
+        help='load test suites from FILE',
+        default='suites.yaml')
+    parser.add_argument(
+        '--ip',
+        help='server ip param for local mode',
+        default='127.0.0.1')
+    parser.add_argument(
+        '--port',
+        help='server port param for local mode',
+        default='19530')
+
+    args = parser.parse_args()
+
+    operations = None
+    # Get all benchmark test suites
+    if args.suites:
+        with open(args.suites) as f:
+            suites_dict = load(f)
+            f.close()
+        # With definition order
+        operations = operations_parser(suites_dict, run_type=args.run_type)
+
+    # init_env()
+    run_params = {"run_count": args.run_count}
+
+    if args.image:
+        # for docker mode
+        if args.local:
+            logger.error("Local mode and docker mode are incompatible arguments")
+            sys.exit(-1)
+        # Docker pull image
+        if not utils.pull_image(args.image):
+            raise Exception('Image %s pull failed' % image)
+
+        # TODO: Check milvus server port is available
+        logger.info("Init: remove all containers created with image: %s" % args.image)
+        utils.remove_all_containers(args.image)
+        runner = DockerRunner(args.image)
+        for operation_type in operations:
+            logger.info("Start run test, test type: %s" % operation_type)
+            run_params["params"] = operations[operation_type]
+            runner.run({operation_type: run_params}, run_type=args.run_type)
+            logger.info("Run params: %s" % str(run_params))
+
+    if args.local:
+        # for local mode
+        ip = args.ip
+        port = args.port
+
+        runner = LocalRunner(ip, port)
+        for operation_type in operations:
+            logger.info("Start run local mode test, test type: %s" % operation_type)
+            run_params["params"] = operations[operation_type]
+            runner.run({operation_type: run_params}, run_type=args.run_type)
+            logger.info("Run params: %s" % str(run_params))
+
+
+if __name__ == "__main__":
+    main()

tests/milvus_benchmark/operation.py

@@ -0,0 +1,10 @@
+from __future__ import absolute_import
+import pdb
+import time
+
+class Base(object):
+    pass
+
+
+class Insert(Base):
+    pass

tests/milvus_benchmark/parser.py

@@ -0,0 +1,66 @@
+import pdb
+import logging
+
+logger = logging.getLogger("milvus_benchmark.parser")
+
+
+def operations_parser(operations, run_type="performance"):
+    definitions = operations[run_type]
+    return definitions
+
+
+def table_parser(table_name):
+    tmp = table_name.split("_")
+    # if len(tmp) != 5:
+    #     return None
+    data_type = tmp[0]
+    table_size_unit = tmp[1][-1]
+    table_size = tmp[1][0:-1]
+    if table_size_unit == "m":
+        table_size = int(table_size) * 1000000
+    elif table_size_unit == "b":
+        table_size = int(table_size) * 1000000000
+    index_file_size = int(tmp[2])
+    dimension = int(tmp[3])
+    metric_type = str(tmp[4])
+    return (data_type, table_size, index_file_size, dimension, metric_type)
+
+
+def search_params_parser(param):
+    # parse top-k, set default value if top-k not in param
+    if "top_ks" not in param:
+        top_ks = [10]
+    else:
+        top_ks = param["top_ks"]
+    if isinstance(top_ks, int):
+        top_ks = [top_ks]
+    elif isinstance(top_ks, list):
+        top_ks = list(top_ks)
+    else:
+        logger.warning("Invalid format top-ks: %s" % str(top_ks))
+
+    # parse nqs, set default value if nq not in param
+    if "nqs" not in param:
+        nqs = [10]
+    else:
+        nqs = param["nqs"]
+    if isinstance(nqs, int):
+        nqs = [nqs]
+    elif isinstance(nqs, list):
+        nqs = list(nqs)
+    else:
+        logger.warning("Invalid format nqs: %s" % str(nqs))
+
+    # parse nprobes
+    if "nprobes" not in param:
+        nprobes = [1]
+    else:
+        nprobes = param["nprobes"]
+    if isinstance(nprobes, int):
+        nprobes = [nprobes]
+    elif isinstance(nprobes, list):
+        nprobes = list(nprobes)
+    else:
+        logger.warning("Invalid format nprobes: %s" % str(nprobes))    
+
+    return top_ks, nqs, nprobes

tests/milvus_benchmark/report.py

@@ -0,0 +1,10 @@
+# from tablereport import Table
+# from tablereport.shortcut import write_to_excel
+
+# RESULT_FOLDER = "results"
+
+# def create_table(headers, bodys, table_name):
+# 	table = Table(header=[headers],
+# 	              body=[bodys])
+
+# 	write_to_excel('%s/%s.xlsx' % (RESULT_FOLDER, table_name), table)

tests/milvus_benchmark/requirements.txt

@@ -0,0 +1,6 @@
+numpy==1.16.3
+pymilvus>=0.1.18
+pyyaml==3.12
+docker==4.0.2
+tableprint==0.8.0
+ansicolors==1.1.8

tests/milvus_benchmark/runner.py

@@ -0,0 +1,219 @@
+import os
+import logging
+import pdb
+import time
+import random
+from multiprocessing import Process
+import numpy as np
+from client import MilvusClient
+import utils
+import parser
+
+logger = logging.getLogger("milvus_benchmark.runner")
+
+SERVER_HOST_DEFAULT = "127.0.0.1"
+SERVER_PORT_DEFAULT = 19530
+VECTORS_PER_FILE = 1000000
+SIFT_VECTORS_PER_FILE = 100000
+MAX_NQ = 10001
+FILE_PREFIX = "binary_"
+
+RANDOM_SRC_BINARY_DATA_DIR = '/tmp/random/binary_data'
+SIFT_SRC_DATA_DIR = '/tmp/sift1b/query'
+SIFT_SRC_BINARY_DATA_DIR = '/tmp/sift1b/binary_data'
+SIFT_SRC_GROUNDTRUTH_DATA_DIR = '/tmp/sift1b/groundtruth'
+
+WARM_TOP_K = 1
+WARM_NQ = 1
+DEFAULT_DIM = 512
+
+GROUNDTRUTH_MAP = {
+    "1000000": "idx_1M.ivecs",
+    "2000000": "idx_2M.ivecs",
+    "5000000": "idx_5M.ivecs",
+    "10000000": "idx_10M.ivecs",
+    "20000000": "idx_20M.ivecs",
+    "50000000": "idx_50M.ivecs",
+    "100000000": "idx_100M.ivecs",
+    "200000000": "idx_200M.ivecs",
+    "500000000": "idx_500M.ivecs",
+    "1000000000": "idx_1000M.ivecs",
+}
+
+
+def gen_file_name(idx, table_dimension, data_type):
+    s = "%05d" % idx
+    fname = FILE_PREFIX + str(table_dimension) + "d_" + s + ".npy"
+    if data_type == "random":
+        fname = RANDOM_SRC_BINARY_DATA_DIR+'/'+fname
+    elif data_type == "sift":
+        fname = SIFT_SRC_BINARY_DATA_DIR+'/'+fname
+    return fname
+
+
+def get_vectors_from_binary(nq, dimension, data_type):
+    # use the first file, nq should be less than VECTORS_PER_FILE
+    if nq > MAX_NQ:
+        raise Exception("Over size nq")
+    if data_type == "random":
+        file_name = gen_file_name(0, dimension, data_type)
+    elif data_type == "sift":
+        file_name = SIFT_SRC_DATA_DIR+'/'+'query.npy'
+    data = np.load(file_name)
+    vectors = data[0:nq].tolist()
+    return vectors
+
+
+class Runner(object):
+    def __init__(self):
+        pass
+
+    def do_insert(self, milvus, table_name, data_type, dimension, size, ni):
+        '''
+        @params:
+            mivlus: server connect instance
+            dimension: table dimensionn
+            # index_file_size: size trigger file merge
+            size: row count of vectors to be insert
+            ni: row count of vectors to be insert each time
+            # store_id: if store the ids returned by call add_vectors or not
+        @return:
+            total_time: total time for all insert operation
+            qps: vectors added per second
+            ni_time: avarage insert operation time
+        '''
+        bi_res = {}
+        total_time = 0.0
+        qps = 0.0
+        ni_time = 0.0
+        if data_type == "random":
+            vectors_per_file = VECTORS_PER_FILE
+        elif data_type == "sift":
+            vectors_per_file = SIFT_VECTORS_PER_FILE
+        if size % vectors_per_file or ni > vectors_per_file:
+            raise Exception("Not invalid table size or ni")
+        file_num = size // vectors_per_file
+        for i in range(file_num):
+            file_name = gen_file_name(i, dimension, data_type)
+            logger.info("Load npy file: %s start" % file_name)
+            data = np.load(file_name)
+            logger.info("Load npy file: %s end" % file_name)
+            loops = vectors_per_file // ni
+            for j in range(loops):
+                vectors = data[j*ni:(j+1)*ni].tolist()
+                ni_start_time = time.time()
+                # start insert vectors
+                start_id = i * vectors_per_file + j * ni
+                end_id = start_id + len(vectors)
+                logger.info("Start id: %s, end id: %s" % (start_id, end_id))
+                ids = [k for k in range(start_id, end_id)]
+                status, ids = milvus.insert(vectors, ids=ids)
+                ni_end_time = time.time()
+                total_time = total_time + ni_end_time - ni_start_time
+
+        qps = round(size / total_time, 2)
+        ni_time = round(total_time / (loops * file_num), 2)
+        bi_res["total_time"] = round(total_time, 2)
+        bi_res["qps"] = qps
+        bi_res["ni_time"] = ni_time
+        return bi_res
+
+    def do_query(self, milvus, table_name, top_ks, nqs, nprobe, run_count):
+        (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+        base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type)
+
+        bi_res = []
+        for index, nq in enumerate(nqs):
+            tmp_res = []
+            for top_k in top_ks:
+                avg_query_time = 0.0
+                total_query_time = 0.0
+                vectors = base_query_vectors[0:nq]
+                logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors)))
+                for i in range(run_count):
+                    logger.info("Start run query, run %d of %s" % (i+1, run_count))
+                    start_time = time.time()
+                    status, query_res = milvus.query(vectors, top_k, nprobe)
+                    total_query_time = total_query_time + (time.time() - start_time)
+                    if status.code:
+                        logger.error("Query failed with message: %s" % status.message)
+                avg_query_time = round(total_query_time / run_count, 2)
+                logger.info("Avarage query time: %.2f" % avg_query_time)
+                tmp_res.append(avg_query_time)
+            bi_res.append(tmp_res)
+        return bi_res
+
+    def do_query_ids(self, milvus, table_name, top_k, nq, nprobe):
+        (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+        base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type)
+        vectors = base_query_vectors[0:nq]
+        logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors)))
+        status, query_res = milvus.query(vectors, top_k, nprobe)
+        if not status.OK():
+            msg = "Query failed with message: %s" % status.message
+            raise Exception(msg)
+        result_ids = []
+        for result in query_res:
+            tmp = []
+            for item in result:
+                tmp.append(item.id)
+            result_ids.append(tmp)
+        return result_ids
+
+    def do_query_acc(self, milvus, table_name, top_k, nq, nprobe, id_store_name):
+        (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name)
+        base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type)
+        vectors = base_query_vectors[0:nq]
+        logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors)))
+        status, query_res = milvus.query(vectors, top_k, nprobe)
+        if not status.OK():
+            msg = "Query failed with message: %s" % status.message
+            raise Exception(msg)
+        # if file existed, cover it
+        if os.path.isfile(id_store_name):
+            os.remove(id_store_name)
+        with open(id_store_name, 'a+') as fd:
+            for nq_item in query_res:
+                for item in nq_item:
+                    fd.write(str(item.id)+'\t')
+                fd.write('\n')
+
+    # compute and print accuracy
+    def compute_accuracy(self, flat_file_name, index_file_name):
+        flat_id_list = []; index_id_list = []
+        logger.info("Loading flat id file: %s" % flat_file_name)
+        with open(flat_file_name, 'r') as flat_id_fd:
+            for line in flat_id_fd:
+                tmp_list = line.strip("\n").strip().split("\t")
+                flat_id_list.append(tmp_list)
+        logger.info("Loading index id file: %s" % index_file_name)
+        with open(index_file_name) as index_id_fd:
+            for line in index_id_fd:
+                tmp_list = line.strip("\n").strip().split("\t")
+                index_id_list.append(tmp_list)
+        if len(flat_id_list) != len(index_id_list):
+            raise Exception("Flat index result length: <flat: %s, index: %s> not match, Acc compute exiting ..." % (len(flat_id_list), len(index_id_list)))
+        # get the accuracy
+        return self.get_recall_value(flat_id_list, index_id_list)
+
+    def get_recall_value(self, flat_id_list, index_id_list):
+        """
+        Use the intersection length
+        """
+        sum_radio = 0.0
+        for index, item in enumerate(index_id_list):
+            tmp = set(item).intersection(set(flat_id_list[index]))
+            sum_radio = sum_radio + len(tmp) / len(item)
+        return round(sum_radio / len(index_id_list), 3)
+
+    """
+    Implementation based on:
+        https://github.com/facebookresearch/faiss/blob/master/benchs/datasets.py
+    """
+    def get_groundtruth_ids(self, table_size):
+        fname = GROUNDTRUTH_MAP[str(table_size)]
+        fname = SIFT_SRC_GROUNDTRUTH_DATA_DIR + "/" + fname
+        a = np.fromfile(fname, dtype='int32')
+        d = a[0]
+        true_ids = a.reshape(-1, d + 1)[:, 1:].copy()
+        return true_ids

tests/milvus_benchmark/suites.yaml

@@ -0,0 +1,38 @@
+# data sets
+datasets:
+  hf5:
+    gist-960,sift-128
+  npy:
+    50000000-512, 100000000-512
+
+operations:
+  # interface: search_vectors
+  query:
+    # dataset: table name you have already created
+    # key starts with "server." need to reconfig and restart server, including nprpbe/nlist/use_blas_threshold/..
+    [
+      # debug
+      # {"dataset": "ip_ivfsq8_1000", "top_ks": [16], "nqs": [1], "server.nprobe": 1, "server.use_blas_threshold": 800, "server.cpu_cache_capacity": 110},
+      
+      {"dataset": "ip_ivfsq8_1000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 1, "server.use_blas_threshold": 800, "server.cpu_cache_capacity": 110},
+      {"dataset": "ip_ivfsq8_1000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 10, "server.use_blas_threshold": 20, "server.cpu_cache_capacity": 110},
+      {"dataset": "ip_ivfsq8_5000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 1, "server.use_blas_threshold": 800, "server.cpu_cache_capacity": 110},
+      {"dataset": "ip_ivfsq8_5000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 10, "server.use_blas_threshold": 20, "server.cpu_cache_capacity": 110},
+      {"dataset": "ip_ivfsq8_40000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 1, "server.use_blas_threshold": 800, "server.cpu_cache_capacity": 110},
+      # {"dataset": "ip_ivfsq8_40000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], "nqs": [1, 10, 100, 1000], "server.nprobe": 10, "server.use_blas_threshold": 20, "server.cpu_cache_capacity": 110},
+    ]
+
+  # interface: add_vectors
+  insert:
+    # index_type: flat/ivf_flat/ivf_sq8
+    [
+      # debug
+
+
+      {"table_name": "ip_ivf_flat_20m_1024", "table.index_type": "ivf_flat", "server.index_building_threshold": 1024, "table.size": 20000000, "table.ni": 100000, "table.dim": 512, "server.cpu_cache_capacity": 110},
+      {"table_name": "ip_ivf_sq8_50m_1024", "table.index_type": "ivf_sq8", "server.index_building_threshold": 1024, "table.size": 50000000, "table.ni": 100000, "table.dim": 512, "server.cpu_cache_capacity": 110},
+    ]
+
+  # TODO: interface: build_index
+  build: []
+

tests/milvus_benchmark/suites_accuracy.yaml

@@ -0,0 +1,121 @@
+
+accuracy:
+  # interface: search_vectors
+  query:
+    [
+      {
+        "dataset": "random_20m_1024_512_ip",
+        # index info
+        "index.index_types": ["flat", "ivf_sq8"],
+        "index.nlists": [16384],
+        "index.metric_types": ["ip"],
+        "nprobes": [1, 16, 64], 
+        "top_ks": [64], 
+        "nqs": [100], 
+        "server.cpu_cache_capacity": 100,
+        "server.resources": ["cpu", "gpu0"],
+        "db_path_prefix": "/test/milvus/db_data/random_20m_1024_512_ip",
+      },
+      # {
+      #   "dataset": "sift_50m_1024_128_l2",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8h"],
+      #   "index.nlists": [16384],
+      #   "index.metric_types": ["l2"],
+      #   "nprobes": [1, 16, 64], 
+      #   "top_ks": [64], 
+      #   "nqs": [100], 
+      #   "server.cpu_cache_capacity": 160,
+      #   "server.resources": ["cpu", "gpu0"],
+      #   "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2",
+      #   "sift_acc": true
+      # },
+      # {
+      #   "dataset": "sift_50m_1024_128_l2",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "index.metric_types": ["l2"],
+      #   "nprobes": [1, 16, 64], 
+      #   "top_ks": [64], 
+      #   "nqs": [100], 
+      #   "server.cpu_cache_capacity": 160,
+      #   "server.resources": ["cpu", "gpu0"],
+      #   "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8",
+      #   "sift_acc": true
+      # },
+      # {
+      #   "dataset": "sift_1b_2048_128_l2",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8h"],
+      #   "index.nlists": [16384],
+      #   "index.metric_types": ["l2"],
+      #   "nprobes": [1, 16, 64, 128], 
+      #   "top_ks": [64], 
+      #   "nqs": [100], 
+      #   "server.cpu_cache_capacity": 200,
+      #   "server.resources": ["cpu"],
+      #   "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h",
+      #   "sift_acc": true
+      # },
+      # {
+      #   "dataset": "sift_1b_2048_128_l2",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8h"],
+      #   "index.nlists": [16384],
+      #   "index.metric_types": ["l2"],
+      #   "nprobes": [1, 16, 64, 128], 
+      #   "top_ks": [64], 
+      #   "nqs": [100], 
+      #   "server.cpu_cache_capacity": 200,
+      #   "server.resources": ["cpu", "gpu0"],
+      #   "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h",
+      #   "sift_acc": true
+      # },
+      # {
+      #   "dataset": "sift_1b_2048_128_l2",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8h"],
+      #   "index.nlists": [16384],
+      #   "index.metric_types": ["l2"],
+      #   "nprobes": [1, 16, 64, 128], 
+      #   "top_ks": [64], 
+      #   "nqs": [100], 
+      #   "server.cpu_cache_capacity": 200,
+      #   "server.resources": ["cpu", "gpu0", "gpu1"],
+      #   "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h",
+      #   "sift_acc": true
+      # },
+      # {
+      #   "dataset": "sift_1m_1024_128_l2", 
+      #   "index.index_types": ["flat", "ivf_sq8"], 
+      #   "index.nlists": [16384],
+      #   "nprobes": [1, 32, 128, 256, 512], 
+      #   "nqs": 10,
+      #   "top_ks": 10, 
+      #   "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 16,
+      # },
+      # {
+      #   "dataset": "sift_10m_1024_128_l2", 
+      #   "index.index_types": ["flat", "ivf_sq8"], 
+      #   "index.nlists": [16384],
+      #   "nprobes": [1, 32, 128, 256, 512], 
+      #   "nqs": 10,
+      #   "top_ks": 10, 
+      #   "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 32,
+      # },
+      # {
+      #   "dataset": "sift_50m_1024_128_l2", 
+      #   "index.index_types": ["flat", "ivf_sq8"], 
+      #   "index.nlists": [16384],
+      #   "nprobes": [1, 32, 128, 256, 512], 
+      #   "nqs": 10,
+      #   "top_ks": 10, 
+      #   "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 64,
+      # }
+
+
+    ]

tests/milvus_benchmark/suites_performance.yaml

@@ -0,0 +1,258 @@
+performance:
+
+  # interface: add_vectors
+  insert:
+    # index_type: flat/ivf_flat/ivf_sq8/mix_nsg
+    [
+      # debug
+      # data_type / data_size / index_file_size / dimension
+      # data_type: random / ann_sift
+      # data_size: 10m / 1b
+      # {
+      #   "table_name": "random_50m_1024_512_ip", 
+      #   "ni_per": 100000, 
+      #   "processes": 5, # multiprocessing
+      #   "server.cpu_cache_capacity": 16,
+      #   # "server.resources": ["gpu0", "gpu1"],
+      #   "db_path_prefix": "/test/milvus/db_data"
+      # },
+      # {
+      #   "table_name": "random_5m_1024_512_ip", 
+      #   "ni_per": 100000, 
+      #   "processes": 5, # multiprocessing
+      #   "server.cpu_cache_capacity": 16,
+      #   "server.resources": ["gpu0", "gpu1"],
+      #   "db_path_prefix": "/test/milvus/db_data/random_5m_1024_512_ip"
+      # },
+      # {
+      #   "table_name": "sift_1m_50_128_l2", 
+      #   "ni_per": 100000, 
+      #   "processes": 5, # multiprocessing
+      #   # "server.cpu_cache_capacity": 16,
+      #   "db_path_prefix": "/test/milvus/db_data"
+      # },
+      # {
+      #   "table_name": "sift_1m_256_128_l2", 
+      #   "ni_per": 100000, 
+      #   "processes": 5, # multiprocessing
+      #   # "server.cpu_cache_capacity": 16,
+      #   "db_path_prefix": "/test/milvus/db_data"
+      # }
+      # {
+      #   "table_name": "sift_50m_1024_128_l2", 
+      #   "ni_per": 100000, 
+      #   "processes": 5, # multiprocessing
+      #   # "server.cpu_cache_capacity": 16,
+      # },
+      # {
+      #   "table_name": "sift_100m_1024_128_l2", 
+      #   "ni_per": 100000, 
+      #   "processes": 5, # multiprocessing
+      # },
+      # {
+      #   "table_name": "sift_1b_2048_128_l2", 
+      #   "ni_per": 100000, 
+      #   "processes": 5, # multiprocessing
+      #   "server.cpu_cache_capacity": 16,
+      # }
+    ]
+
+  # interface: search_vectors
+  query:
+    # dataset: table name you have already created
+    # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
+    [
+      # {
+      #   "dataset": "sift_1b_2048_128_l2",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8h"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [8, 32], 
+      #   "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
+      #   "nqs": [1, 10, 100, 500, 1000], 
+      #   "processes": 1, # multiprocessing
+      #   "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 200,
+      #   "server.resources": ["cpu", "gpu0"],
+      #   "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h"
+      # },
+      # {
+      #   "dataset": "sift_1b_2048_128_l2",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [8, 32], 
+      #   "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
+      #   "nqs": [1, 10, 100, 500, 1000], 
+      #   "processes": 1, # multiprocessing
+      #   "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 200,
+      #   "server.resources": ["cpu", "gpu0"],
+      #   "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2"
+      # },
+      # {
+      #   "dataset": "sift_1b_2048_128_l2",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8h"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [8, 32], 
+      #   "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
+      #   "nqs": [1, 10, 100, 500, 1000], 
+      #   "processes": 1, # multiprocessing
+      #   "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 200,
+      #   "server.resources": ["cpu"],
+      #   "db_path_prefix": "/test/milvus/db_data"
+      # },
+      {
+        "dataset": "random_50m_1024_512_ip",
+        "index.index_types": ["ivf_sq8h"],
+        "index.nlists": [16384],
+        "nprobes": [8], 
+        # "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
+        "top_ks": [512], 
+        # "nqs": [1, 10, 100, 500, 1000], 
+        "nqs": [500], 
+        "server.use_blas_threshold": 1100, 
+        "server.cpu_cache_capacity": 150,
+        "server.gpu_cache_capacity": 6,
+        "server.resources": ["cpu", "gpu0", "gpu1"],
+        "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip"
+      },
+      # {
+      #   "dataset": "random_50m_1024_512_ip",
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [8, 32], 
+      #   "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
+      #   "nqs": [1, 10, 100, 500, 1000], 
+      #   "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 150,
+      #   "server.resources": ["cpu", "gpu0", "gpu1"],
+      #   "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_sq8"
+      # },
+      # {
+      #   "dataset": "random_20m_1024_512_ip",
+      #   "index.index_types": ["flat"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [50], 
+      #   "top_ks": [64], 
+      #   "nqs": [10], 
+      #   "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 100,
+      #   "server.resources": ["cpu", "gpu0", "gpu1"],
+      #   "db_path_prefix": "/test/milvus/db_data/random_20m_1024_512_ip"
+      # },
+      # {
+      #   "dataset": "random_100m_1024_512_ip",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [8, 32], 
+      #   "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
+      #   "nqs": [1, 10, 100, 500, 1000], 
+      #   "processes": 1, # multiprocessing
+      #   "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 250,
+      #   "server.resources": ["cpu", "gpu0"],
+      #   "db_path_prefix": "/test/milvus/db_data"
+      # },
+      # {
+      #   "dataset": "random_100m_1024_512_ip",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [8, 32], 
+      #   "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], 
+      #   "nqs": [1, 10, 100, 500, 1000], 
+      #   "processes": 1, # multiprocessing
+      #   "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 250,
+      #   "server.resources": ["cpu"],
+      #   "db_path_prefix": "/test/milvus/db_data"
+      # },
+      # {
+      #   "dataset": "random_10m_1024_512_ip",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [1], 
+      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], 
+      #   "nqs": [1, 10, 100, 500, 800], 
+      #   "processes": 1, # multiprocessing
+      #   # "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 16,
+      # },
+      # {
+      #   "dataset": "random_10m_1024_512_l2",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [1], 
+      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], 
+      #   "nqs": [1, 10, 100, 500, 800], 
+      #   "processes": 1, # multiprocessing
+      #   "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 64
+      # },
+      # {
+      #   "dataset": "sift_500m_1024_128_l2",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [1], 
+      #   "top_ks": [1, 8, 16, 64, 256, 512, 1000], 
+      #   "nqs": [1, 100, 500, 800, 1000, 1500], 
+      #   # "top_ks": [256],
+      #   # "nqs": [800],
+      #   "processes": 1, # multiprocessing
+      #   # "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 120,
+      #   "server.resources": ["gpu0", "gpu1"],
+      #   "db_path_prefix": "/test/milvus/db_data"
+      # },
+      # {
+      #    "dataset": "sift_1b_2048_128_l2",
+      #    # index info
+      #    "index.index_types": ["ivf_sq8h"],
+      #    "index.nlists": [16384],
+      #    "nprobes": [1], 
+      #    # "top_ks": [1], 
+      #    # "nqs": [1], 
+      #    "top_ks": [256],
+      #    "nqs": [800],
+      #    "processes": 1, # multiprocessing
+      #    "server.use_blas_threshold": 1100, 
+      #    "server.cpu_cache_capacity": 110,
+      #    "server.resources": ["cpu", "gpu0"],
+      #    "db_path_prefix": "/test/milvus/db_data"
+      # },
+      # {
+      #   "dataset": "random_50m_1024_512_l2",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [1], 
+      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], 
+      #   "nqs": [1, 10, 100, 500, 800], 
+      #   # "top_ks": [256],
+      #   # "nqs": [800],
+      #   "processes": 1, # multiprocessing
+      #   "server.use_blas_threshold": 1100, 
+      #   "server.cpu_cache_capacity": 128
+      # },
+    # [
+    #   {
+    #     "dataset": "sift_1m_50_128_l2",
+    #     # index info
+    #     "index.index_types": ["ivf_sq8"],
+    #     "index.nlists": [16384],
+    #     "nprobes": [1], 
+    #     "top_ks": [1], 
+    #     "nqs": [1], 
+    #     "db_path_prefix": "/test/milvus/db_data"
+    #     # "processes": 1, # multiprocessing
+    #     # "server.use_blas_threshold": 1100, 
+    #     # "server.cpu_cache_capacity": 256
+    #   }
+    ]

tests/milvus_benchmark/suites_stability.yaml

@@ -0,0 +1,17 @@
+
+stability:
+  # interface: search_vectors / add_vectors mix operation
+  query:
+    [
+      {
+        "dataset": "random_20m_1024_512_ip",
+        # "nqs": [1, 10, 100, 1000, 10000],
+        # "pds": [0.1, 0.44, 0.44, 0.02],
+        "query_process_num": 10,
+        # each 10s, do an insertion
+        # "insert_interval": 1,
+        # minutes
+        "during_time": 360,
+        "server.cpu_cache_capacity": 100
+      },
+    ]

tests/milvus_benchmark/suites_yzb.yaml

@@ -0,0 +1,171 @@
+#"server.resources": ["gpu0", "gpu1"]
+
+performance:
+  # interface: search_vectors
+  query:
+    # dataset: table name you have already created
+    # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity ..
+    [
+      # debug
+      # {
+      #   "dataset": "random_10m_1024_512_ip",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [1],
+      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
+      #   "nqs": [1, 10, 100, 500, 800],
+      #   "processes": 1, # multiprocessing
+      #   "server.use_blas_threshold": 1100,
+      #   "server.cpu_cache_capacity": 16,
+      # },
+      # {
+      #   "dataset": "random_10m_1024_512_ip",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [1],
+      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
+      #   "nqs": [1, 10, 100, 500, 800],
+      #   "processes": 1, # multiprocessing
+      #   "server.use_blas_threshold": 1100,
+      #   "server.cpu_cache_capacity": 16,
+      # },
+      # {
+      #   "dataset": "random_10m_1024_512_ip",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [1],
+      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
+      #   "nqs": [1, 10, 100, 500, 800],
+      #   "processes": 1, # multiprocessing
+      #   # "server.use_blas_threshold": 1100,
+      #   "server.cpu_cache_capacity": 16,
+      # },
+      # {
+      #   "dataset": "random_10m_1024_512_ip",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [1],
+      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
+      #   "nqs": [1, 10, 100, 500, 800],
+      #   "processes": 1, # multiprocessing
+      #   # "server.use_blas_threshold": 1100,
+      #   "server.cpu_cache_capacity": 16,
+      # },
+      # {
+      #   "dataset": "random_10m_1024_512_l2",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [1],
+      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
+      #   "nqs": [1, 10, 100, 500, 800],
+      #   "processes": 1, # multiprocessing
+      #   "server.use_blas_threshold": 1100,
+      #   "server.cpu_cache_capacity": 64
+      # },
+#       {
+#        "dataset": "sift_50m_1024_128_l2",
+#        # index info
+#        "index.index_types": ["ivf_sq8"],
+#        "index.nlists": [16384],
+#        "nprobes": [1, 32, 128],
+#        "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
+#        "nqs": [1, 10, 100, 500, 800],
+#        # "top_ks": [256],
+#        # "nqs": [800],
+#        "processes": 1, # multiprocessing
+#         "server.use_blas_threshold": 1100,
+#         "server.cpu_cache_capacity": 310,
+#        "server.resources": ["gpu0", "gpu1"]
+#       },
+       {
+        "dataset": "sift_1m_1024_128_l2",
+        # index info
+        "index.index_types": ["ivf_sq8"],
+        "index.nlists": [16384],
+        "nprobes": [32],
+        "top_ks": [10],
+        "nqs": [100],
+        # "top_ks": [256],
+        # "nqs": [800],
+        "processes": 1, # multiprocessing
+         "server.use_blas_threshold": 1100,
+         "server.cpu_cache_capacity": 310,
+        "server.resources": ["cpu"]
+       },
+       {
+        "dataset": "sift_1m_1024_128_l2",
+        # index info
+        "index.index_types": ["ivf_sq8"],
+        "index.nlists": [16384],
+        "nprobes": [32],
+        "top_ks": [10],
+        "nqs": [100],
+        # "top_ks": [256],
+        # "nqs": [800],
+        "processes": 1, # multiprocessing
+         "server.use_blas_threshold": 1100,
+         "server.cpu_cache_capacity": 310,
+        "server.resources": ["gpu0"]
+       },
+       {
+        "dataset": "sift_1m_1024_128_l2",
+        # index info
+        "index.index_types": ["ivf_sq8"],
+        "index.nlists": [16384],
+        "nprobes": [32],
+        "top_ks": [10],
+        "nqs": [100],
+        # "top_ks": [256],
+        # "nqs": [800],
+        "processes": 1, # multiprocessing
+         "server.use_blas_threshold": 1100,
+         "server.cpu_cache_capacity": 310,
+        "server.resources": ["gpu0", "gpu1"]
+       },
+      # {
+      #    "dataset": "sift_1b_2048_128_l2",
+      #    # index info
+      #    "index.index_types": ["ivf_sq8"],
+      #    "index.nlists": [16384],
+      #    "nprobes": [1],
+      #    "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
+      #    "nqs": [1, 10, 100, 500, 800],
+      #     # "top_ks": [256],
+      #     #   "nqs": [800],
+      #    "processes": 1, # multiprocessing
+      #    "server.use_blas_threshold": 1100,
+      #    "server.cpu_cache_capacity": 310
+      # },
+#       {
+#         "dataset": "random_50m_1024_512_l2",
+#         # index info
+#         "index.index_types": ["ivf_sq8"],
+#         "index.nlists": [16384],
+#         "nprobes": [1],
+#         "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
+#         "nqs": [1, 10, 100, 500, 800],
+#         # "top_ks": [256],
+#         # "nqs": [800],
+#         "processes": 1, # multiprocessing
+#         "server.use_blas_threshold": 1100,
+#         "server.cpu_cache_capacity": 128,
+#         "server.resources": ["gpu0", "gpu1"]
+#       },
+      # {
+      #   "dataset": "random_100m_1024_512_ip",
+      #   # index info
+      #   "index.index_types": ["ivf_sq8"],
+      #   "index.nlists": [16384],
+      #   "nprobes": [1],
+      #   "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256],
+      #   "nqs": [1, 10, 100, 500, 800],
+      #   "processes": 1, # multiprocessing
+      #   "server.use_blas_threshold": 1100,
+      #   "server.cpu_cache_capacity": 256
+      # },
+    ]

tests/milvus_benchmark/utils.py

@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function
+
+__true_print = print  # noqa
+
+import os
+import sys
+import pdb
+import time
+import datetime
+import argparse
+import threading
+import logging
+import docker
+import multiprocessing
+import numpy
+# import psutil
+from yaml import load, dump
+import tableprint as tp
+
+logger = logging.getLogger("milvus_benchmark.utils")
+
+MULTI_DB_SLAVE_PATH = "/opt/milvus/data2;/opt/milvus/data3"
+
+
+def get_current_time():
+    return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
+
+
+def print_table(headers, columns, data):
+    bodys = []
+    for index, value in enumerate(columns):
+        tmp = [value]
+        tmp.extend(data[index])
+        bodys.append(tmp)
+    tp.table(bodys, headers)
+
+
+def modify_config(k, v, type=None, file_path="conf/server_config.yaml", db_slave=None):
+    if not os.path.isfile(file_path):
+        raise Exception('File: %s not found' % file_path)
+    with open(file_path) as f:
+        config_dict = load(f)
+        f.close()
+    if config_dict:
+        if k.find("use_blas_threshold") != -1:
+            config_dict['engine_config']['use_blas_threshold'] = int(v)
+        elif k.find("cpu_cache_capacity") != -1:
+            config_dict['cache_config']['cpu_cache_capacity'] = int(v)
+        elif k.find("gpu_cache_capacity") != -1:
+            config_dict['cache_config']['gpu_cache_capacity'] = int(v)
+        elif k.find("resource_pool") != -1:
+            config_dict['resource_config']['resource_pool'] = v
+
+        if db_slave:
+            config_dict['db_config']['db_slave_path'] = MULTI_DB_SLAVE_PATH
+        with open(file_path, 'w') as f:
+            dump(config_dict, f, default_flow_style=False)
+        f.close()
+    else:
+        raise Exception('Load file:%s error' % file_path)
+
+
+def pull_image(image):
+    registry = image.split(":")[0]
+    image_tag = image.split(":")[1]
+    client = docker.APIClient(base_url='unix://var/run/docker.sock')
+    logger.info("Start pulling image: %s" % image)
+    return client.pull(registry, image_tag)
+
+
+def run_server(image, mem_limit=None, timeout=30, test_type="local", volume_name=None, db_slave=None):
+    import colors
+
+    client = docker.from_env()
+    # if mem_limit is None:
+    #     mem_limit = psutil.virtual_memory().available
+    # logger.info('Memory limit:', mem_limit)
+    # cpu_limit = "0-%d" % (multiprocessing.cpu_count() - 1)
+    # logger.info('Running on CPUs:', cpu_limit)
+    for dir_item in ['logs', 'db']:
+        try:
+            os.mkdir(os.path.abspath(dir_item))
+        except Exception as e:
+            pass
+
+    if test_type == "local":
+        volumes = {
+            os.path.abspath('conf'):
+                {'bind': '/opt/milvus/conf', 'mode': 'ro'},
+            os.path.abspath('logs'):
+                {'bind': '/opt/milvus/logs', 'mode': 'rw'},
+            os.path.abspath('db'):
+                {'bind': '/opt/milvus/db', 'mode': 'rw'},
+        }
+    elif test_type == "remote":
+        if volume_name is None:
+            raise Exception("No volume name")
+        remote_log_dir = volume_name+'/logs'
+        remote_db_dir = volume_name+'/db'
+
+        for dir_item in [remote_log_dir, remote_db_dir]:
+            if not os.path.isdir(dir_item):
+                os.makedirs(dir_item, exist_ok=True)
+        volumes = {
+            os.path.abspath('conf'):
+                {'bind': '/opt/milvus/conf', 'mode': 'ro'},
+            remote_log_dir:
+                {'bind': '/opt/milvus/logs', 'mode': 'rw'},
+            remote_db_dir:
+                {'bind': '/opt/milvus/db', 'mode': 'rw'}
+        }
+        # add volumes
+        if db_slave and isinstance(db_slave, int):
+            for i in range(2, db_slave+1):
+                remote_db_dir = volume_name+'/data'+str(i)
+                if not os.path.isdir(remote_db_dir):
+                    os.makedirs(remote_db_dir, exist_ok=True)
+                volumes[remote_db_dir] = {'bind': '/opt/milvus/data'+str(i), 'mode': 'rw'}
+
+    container = client.containers.run(
+        image,
+        volumes=volumes,
+        runtime="nvidia",
+        ports={'19530/tcp': 19530, '8080/tcp': 8080},
+        environment=["OMP_NUM_THREADS=48"],
+        # cpuset_cpus=cpu_limit,
+        # mem_limit=mem_limit,
+        # environment=[""],
+        detach=True)
+
+    def stream_logs():
+        for line in container.logs(stream=True):
+            logger.info(colors.color(line.decode().rstrip(), fg='blue'))
+
+    if sys.version_info >= (3, 0):
+        t = threading.Thread(target=stream_logs, daemon=True)
+    else:
+        t = threading.Thread(target=stream_logs)
+        t.daemon = True
+    t.start()
+
+    logger.info('Container: %s started' % container)
+    return container
+    # exit_code = container.wait(timeout=timeout)
+    # # Exit if exit code
+    # if exit_code == 0:
+    #     return container
+    # elif exit_code is not None:
+    #     print(colors.color(container.logs().decode(), fg='red'))
+    #     raise Exception('Child process raised exception %s' % str(exit_code))
+
+def restart_server(container):
+    client = docker.APIClient(base_url='unix://var/run/docker.sock')
+
+    client.restart(container.name)
+    logger.info('Container: %s restarted' % container.name)
+    return container
+
+
+def remove_container(container):
+    container.remove(force=True)
+    logger.info('Container: %s removed' % container)
+
+
+def remove_all_containers(image):
+    client = docker.from_env()
+    try:
+        for container in client.containers.list():
+            if image in container.image.tags:
+                container.stop(timeout=30)
+                container.remove(force=True)
+    except Exception as e:
+        logger.error("Containers removed failed")
+
+
+def container_exists(image):
+    '''
+    Check if container existed with the given image name
+    @params: image name
+    @return: container if exists
+    '''
+    res = False
+    client = docker.from_env()
+    for container in client.containers.list():
+        if image in container.image.tags:
+            # True
+            res = container
+    return res
+
+
+if __name__ == '__main__':
+    # print(pull_image('branch-0.3.1-debug'))
+    stop_server()

tests/milvus_python_test/.dockerignore

@@ -0,0 +1,14 @@
+node_modules
+npm-debug.log
+Dockerfile*
+docker-compose*
+.dockerignore
+.git
+.gitignore
+.env
+*/bin
+*/obj
+README.md
+LICENSE
+.vscode
+__pycache__

tests/milvus_python_test/.gitignore

@@ -0,0 +1,13 @@
+.python-version
+.pytest_cache
+__pycache__
+.vscode
+.idea
+
+test_out/
+*.pyc
+
+db/
+logs/
+
+.coverage

tests/milvus_python_test/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.6.8-jessie
+
+LABEL Name=megasearch_engine_test Version=0.0.1
+
+WORKDIR /app
+ADD . /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libc-dev build-essential && \
+    python3 -m pip install -r requirements.txt && \
+    apt-get remove --purge -y 
+
+ENTRYPOINT [ "/app/docker-entrypoint.sh" ]
+CMD [ "start" ]

tests/milvus_python_test/MilvusCases.md

@@ -0,0 +1,143 @@
+# Milvus test cases
+
+## * Interfaces test
+
+### 1. 连接测试
+
+#### 1.1 连接
+
+| cases            | expected                                     |
+| ---------------- | -------------------------------------------- |
+| 非法IP 123.0.0.2 | method: connect raise error in given timeout |
+| 正常 uri         | attr: connected assert true                  |
+| 非法 uri         | method: connect raise error in given timeout |
+| 最大连接数       | all connection attrs: connected assert true  |
+|                  |                                              |
+
+#### 1.2 断开连接
+
+| cases                    | expected            |
+| ------------------------ | ------------------- |
+| 正常连接下，断开连接     | connect raise error |
+| 正常连接下，重复断开连接 | connect raise error |
+
+### 2. Table operation
+
+#### 2.1 表创建
+
+##### 2.1.1 表名
+
+| cases                     | expected    |
+| ------------------------- | ----------- |
+| 基础功能，参数正常        | status pass |
+| 表名已存在                | status fail |
+| 表名："中文"              | status pass |
+| 表名带特殊字符: "-39fsd-" | status pass |
+| 表名带空格: "test1 2"     | status pass |
+| invalid dim: 0            | raise error |
+| invalid dim: -1           | raise error |
+| invalid dim: 100000000    | raise error |
+| invalid dim: "string"     | raise error |
+| index_type: 0             | status pass |
+| index_type: 1             | status pass |
+| index_type: 2             | status pass |
+| index_type: string        | raise error |
+|                           |             |
+
+##### 2.1.2 维数支持
+
+| cases                 | expected    |
+| --------------------- | ----------- |
+| 维数: 0               | raise error |
+| 维数负数: -1          | raise error |
+| 维数最大值: 100000000 | raise error |
+| 维数字符串: "string"  | raise error |
+|                       |             |
+
+##### 2.1.3 索引类型支持
+
+| cases            | expected    |
+| ---------------- | ----------- |
+| 索引类型: 0      | status pass |
+| 索引类型: 1      | status pass |
+| 索引类型: 2      | status pass |
+| 索引类型: string | raise error |
+|                  |             |
+
+#### 2.2 表说明
+
+| cases                  | expected                         |
+| ---------------------- | -------------------------------- |
+| 创建表后，执行describe | 返回结构体，元素与创建表参数一致 |
+|                        |                                  |
+
+#### 2.3 表删除
+
+| cases          | expected               |
+| -------------- | ---------------------- |
+| 删除已存在表名 | has_table return False |
+| 删除不存在表名 | status fail            |
+|                |                        |
+
+#### 2.4 表是否存在
+
+| cases                   | expected     |
+| ----------------------- | ------------ |
+| 存在表，调用has_table   | assert true  |
+| 不存在表，调用has_table | assert false |
+|                         |              |
+
+#### 2.5 查询表记录条数
+
+| cases                | expected                 |
+| -------------------- | ------------------------ |
+| 空表                 | 0                        |
+| 空表插入数据（单条） | 1                        |
+| 空表插入数据（多条） | assert length of vectors |
+
+#### 2.6 查询表数量
+
+| cases                                         | expected                         |
+| --------------------------------------------- | -------------------------------- |
+| 两张表，一张空表，一张有数据：调用show tables | assert length of table list == 2 |
+|                                               |                                  |
+
+### 3. Add vectors
+
+| interfaces  | cases                                                     | expected                             |
+| ----------- | --------------------------------------------------------- | ------------------------------------ |
+| add_vectors | add basic                                                 | assert length of ids == nq           |
+|             | add vectors into table not existed                        | status fail                          |
+|             | dim not match: single vector                              | status fail                          |
+|             | dim not match: vector list                                | status fail                          |
+|             | single vector element empty                               | status fail                          |
+|             | vector list element empty                                 | status fail                          |
+|             | query immediately after adding                            | status pass                          |
+|             | query immediately after sleep 6s                          | status pass && length of result == 1 |
+|             | concurrent add with multi threads(share one connection)   | status pass                          |
+|             | concurrent add with multi threads(independent connection) | status pass                          |
+|             | concurrent add with multi process(independent connection) | status pass                          |
+|             | index_type: 2                                             | status pass                          |
+|             | index_type: string                                        | raise error                          |
+|             |                                                           |                                      |
+
+### 4. Search vectors
+
+| interfaces     | cases                                             | expected                         |
+| -------------- | ------------------------------------------------- | -------------------------------- |
+| search_vectors | search basic(query vector in vectors, top-k<nq)   | assert length of result == nq    |
+|                | search vectors into table not existed             | status fail                      |
+|                | basic top-k                                       | score of query vectors  == 100.0 |
+|                | invalid top-k: 0                                  | raise error                      |
+|                | invalid top-k: -1                                 | raise error                      |
+|                | invalid top-k: "string"                           | raise error                      |
+|                | top-k > nq                                        | assert length of result == nq    |
+|                | concurrent search                                 | status pass                      |
+|                | query_range(get_current_day(), get_current_day()) | assert length of result == nq    |
+|                | invalid query_range: ""                           | raise error                      |
+|                | query_range(get_last_day(2), get_last_day(1))     | assert length of result == 0     |
+|                | query_range(get_last_day(2), get_current_day())   | assert length of result == nq    |
+|                | query_range((get_last_day(2), get_next_day(2))    | assert length of result == nq    |
+|                | query_range((get_current_day(), get_next_day(2))  | assert length of result == nq    |
+|                | query_range(get_next_day(1), get_next_day(2))     | assert length of result == 0     |
+|                | score: vector[i] = vector[i]+-0.01                | score > 99.9                     |

tests/milvus_python_test/README.md

@@ -0,0 +1,23 @@
+# Requirements
+* python 3.6.8+
+* pip install -r requirements.txt
+
+# How to use this Test Project
+```shell
+pytest . --level=1
+```
+or test connect function only
+
+```shell
+pytest test_connect.py --level=1
+```
+
+with allure test report
+
+ ```shell
+pytest --alluredir=test_out . -q -v
+allure serve test_out
+ ```
+# Contribution getting started
+* Follow PEP-8 for naming and black for formatting.
+

tests/milvus_python_test/conftest.py

@@ -0,0 +1,132 @@
+import socket 
+import pdb
+import logging
+
+import pytest
+from utils import gen_unique_str
+from milvus import Milvus, IndexType, MetricType
+
+index_file_size = 10
+
+
+def pytest_addoption(parser):
+    parser.addoption("--ip", action="store", default="localhost")
+    parser.addoption("--port", action="store", default=19530)
+    parser.addoption("--internal", action="store", default=False)
+
+
+def check_server_connection(request):
+    ip = request.config.getoption("--ip")
+    port = request.config.getoption("--port")
+    connected = True
+    if ip and (ip not in ['localhost', '127.0.0.1']):
+        try:
+            socket.getaddrinfo(ip, port, 0, 0, socket.IPPROTO_TCP) 
+        except Exception as e:
+            print("Socket connnet failed: %s" % str(e))
+            connected = False
+    return connected
+
+
+def get_args(request):
+    args = {
+        "ip": request.config.getoption("--ip"),
+        "port": request.config.getoption("--port")
+    }
+    return args
+
+
+@pytest.fixture(scope="module")
+def connect(request):
+    ip = request.config.getoption("--ip")
+    port = request.config.getoption("--port")
+    milvus = Milvus()
+    try:
+        milvus.connect(host=ip, port=port)
+    except:
+        pytest.exit("Milvus server can not connected, exit pytest ...")
+
+    def fin():
+        try:
+            milvus.disconnect()
+        except:
+            pass
+
+    request.addfinalizer(fin)
+    return milvus
+
+
+@pytest.fixture(scope="module")
+def dis_connect(request):
+    ip = request.config.getoption("--ip")
+    port = request.config.getoption("--port")
+    milvus = Milvus()
+    milvus.connect(host=ip, port=port)
+    milvus.disconnect()
+    def fin():
+        try:
+            milvus.disconnect()
+        except:
+            pass
+
+    request.addfinalizer(fin)
+    return milvus
+
+
+@pytest.fixture(scope="module")
+def args(request):
+    ip = request.config.getoption("--ip")
+    port = request.config.getoption("--port")
+    internal = request.config.getoption("--internal")
+    args = {"ip": ip, "port": port}
+    if internal:
+        args = {"ip": ip, "port": port, "internal": internal}
+    return args
+
+
+@pytest.fixture(scope="function")
+def table(request, connect):
+    ori_table_name = getattr(request.module, "table_id", "test")
+    table_name = gen_unique_str(ori_table_name)
+    dim = getattr(request.module, "dim", "128")
+    param = {'table_name': table_name,
+             'dimension': dim,
+             'index_file_size': index_file_size,
+             'metric_type': MetricType.L2}
+    status = connect.create_table(param)
+    # logging.getLogger().info(status)
+    if not status.OK():
+        pytest.exit("Table can not be created, exit pytest ...")
+
+    def teardown():
+        status, table_names = connect.show_tables()
+        for table_name in table_names:
+            connect.delete_table(table_name)
+
+    request.addfinalizer(teardown)
+
+    return table_name
+
+
+@pytest.fixture(scope="function")
+def ip_table(request, connect):
+    ori_table_name = getattr(request.module, "table_id", "test")
+    table_name = gen_unique_str(ori_table_name)
+    dim = getattr(request.module, "dim", "128")
+    param = {'table_name': table_name,
+             'dimension': dim,
+             'index_file_size': index_file_size,
+             'metric_type': MetricType.IP}
+    status = connect.create_table(param)
+    # logging.getLogger().info(status)
+    if not status.OK():
+        pytest.exit("Table can not be created, exit pytest ...")
+
+    def teardown():
+        status, table_names = connect.show_tables()
+        for table_name in table_names:
+            connect.delete_table(table_name)
+
+    request.addfinalizer(teardown)
+
+    return table_name