Merge remote-tracking branch 'upstream/branch-0.4.0' into branch-0.4.0

Former-commit-id: be07f42345fd96adf2aa13588b45e6ada88eb7d4

ci/jenkinsfile/cleanup_staging.groovy

@@ -0,0 +1,13 @@
+try {
+    def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
+    if (!result) {
+        sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
+    }
+} catch (exc) {
+    def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
+    if (!result) {
+        sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
+    }
+    throw exc
+}
+

ci/jenkinsfile/deploy2staging.groovy

@@ -0,0 +1,16 @@
+try {
+    sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+    sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus'
+    sh 'helm repo update'
+    dir ("milvus-helm") {
+        checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
+        dir ("milvus/milvus-gpu") {
+            sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.4.0 ."
+        }
+    }
+} catch (exc) {
+    echo 'Helm running failed!'
+    sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
+    throw exc
+}
+

ci/jenkinsfile/dev_test.groovy

@@ -3,7 +3,7 @@ timeout(time: 40, unit: 'MINUTES') {
         dir ("${PROJECT_NAME}_test") {
             checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
             sh 'python3 -m pip install -r requirements.txt'
-            sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local"
+            sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local"
         }
 
         // mysql database backend test
@@ -16,11 +16,11 @@ timeout(time: 40, unit: 'MINUTES') {
         }
         dir ("milvus-helm") {
             dir ("milvus/milvus-gpu") {
-                sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.3.1 ."
+                sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ."
             }
         }
         dir ("${PROJECT_NAME}_test") {
-            sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local"
+            sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local"
         }
     } catch (exc) {
         echo 'Milvus Test Failed !'

ci/jenkinsfile/nightly_publish_docker.groovy

@@ -0,0 +1,38 @@
+container('publish-docker') {
+    timeout(time: 15, unit: 'MINUTES') {
+        gitlabCommitStatus(name: 'Publish Engine Docker') {
+            try {
+                dir ("${PROJECT_NAME}_build") {
+                    checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
+                    dir ("docker/deploy/ubuntu16.04/free_version") {
+                        sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz"
+                        sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz"
+                        try {
+                            def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}")
+                            docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_PUBLISH_USER}") {
+                                customImage.push()
+                            }
+                            docker.withRegistry('https://zilliz.azurecr.cn', "${params.AZURE_DOCKER_PUBLISH_USER}") {
+                                customImage.push()
+                            }
+                            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
+                                updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'success'
+                                echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}"
+                            }
+                        } catch (exc) {
+                            updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled'
+                            throw exc
+                        } finally {
+                            sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}"
+                        }
+                    }
+                }
+            } catch (exc) {
+                updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed'
+                echo 'Publish docker failed!'
+                throw exc
+            }
+        }
+    }
+}
+

ci/jenkinsfile/publish_docker.groovy

@@ -8,8 +8,8 @@ container('publish-docker') {
                         sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz"
                         sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz"
                         try {
+                            def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}")
                             docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_PUBLISH_USER}") {
-                                def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}")
                                 customImage.push()
                             }
                             if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {

ci/jenkinsfile/staging_test.groovy

@@ -0,0 +1,31 @@
+timeout(time: 40, unit: 'MINUTES') {
+    try {
+        dir ("${PROJECT_NAME}_test") {
+            checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
+            sh 'python3 -m pip install -r requirements.txt'
+            def service_ip = sh (script: "kubectl get svc --namespace milvus-1 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim()
+            sh "pytest . --alluredir=\"test_out/staging/single/sqlite\" --ip ${service_ip}"
+        }
+
+        // mysql database backend test
+        load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
+
+        if (!fileExists('milvus-helm')) {
+            dir ("milvus-helm") {
+                checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
+            }
+        }
+        dir ("milvus-helm") {
+            dir ("milvus/milvus-gpu") {
+                sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ."
+            }
+        }
+        dir ("${PROJECT_NAME}_test") {
+            def service_ip = sh (script: "kubectl get svc --namespace milvus-2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim()
+            sh "pytest . --alluredir=\"test_out/staging/single/mysql\" --ip ${service_ip}"
+        }
+    } catch (exc) {
+        echo 'Milvus Test Failed !'
+        throw exc
+    }
+}

ci/jenkinsfile/upload_dev_test_out.groovy

@@ -1,8 +1,8 @@
 timeout(time: 5, unit: 'MINUTES') {
     dir ("${PROJECT_NAME}_test") {
-        if (fileExists('test_out')) {
+        if (fileExists('test_out/dev')) {
             def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
-            fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
+            fileTransfer.FileTransfer("test_out/dev/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
             if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
                 echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
             }

ci/jenkinsfile/upload_staging_test_out.groovy

@@ -0,0 +1,13 @@
+timeout(time: 5, unit: 'MINUTES') {
+    dir ("${PROJECT_NAME}_test") {
+        if (fileExists('test_out/staging')) {
+            def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
+            fileTransfer.FileTransfer("test_out/staging/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
+            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
+                echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
+            }
+        } else {
+            error("Milvus Dev Test Out directory don't exists!")
+        }
+    }
+}

ci/nightly_main_jenkinsfile

@@ -106,7 +106,7 @@ spec:
                             steps {
                                 gitlabCommitStatus(name: 'Publish Docker') {
                                     script {
-                                        load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy"
+                                        load "${env.WORKSPACE}/ci/jenkinsfile/nightly_publish_docker.groovy"
                                     }
                                 }
                             }
@@ -208,17 +208,17 @@ spec:
                                 }
                                 success {
                                     script {
-                                        echo "Milvus Single Node CI/CD success !"
+                                        echo "Milvus Deploy to Dev Single Node CI/CD success !"
                                     }
                                 }
                                 aborted {
                                     script {
-                                        echo "Milvus Single Node CI/CD aborted !"
+                                        echo "Milvus Deploy to Dev Single Node CI/CD aborted !"
                                     }
                                 }
                                 failure {
                                     script {
-                                        echo "Milvus Single Node CI/CD failure !"
+                                        echo "Milvus Deploy to Dev Single Node CI/CD failure !"
                                     }
                                 }
                             }
@@ -300,17 +300,114 @@ spec:
                                 }
                                 success {
                                     script {
-                                        echo "Milvus Cluster CI/CD success !"
+                                        echo "Milvus Deploy to Dev Cluster CI/CD success !"
                                     }
                                 }
                                 aborted {
                                     script {
-                                        echo "Milvus Cluster CI/CD aborted !"
+                                        echo "Milvus Deploy to Dev Cluster CI/CD aborted !"
                                     }
                                 }
                                 failure {
                                     script {
-                                        echo "Milvus Cluster CI/CD failure !"
+                                        echo "Milvus Deploy to Dev Cluster CI/CD failure !"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                stage("Deploy to Staging") {
+                    parallel {
+                        stage("Single Node") {
+                            agent {
+                                kubernetes {
+                                    label 'dev-test'
+                                    defaultContainer 'jnlp'
+                                    yaml """
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus
+    componet: test
+spec:
+  containers:
+  - name: milvus-testframework
+    image: registry.zilliz.com/milvus/milvus-test:v0.2
+    command:
+    - cat
+    tty: true
+    volumeMounts:
+    - name: kubeconf
+      mountPath: /root/.kube/
+      readOnly: true
+  volumes:
+  - name: kubeconf
+    secret:
+      secretName: aks-gpu-cluster-config
+"""
+                                }
+                            }
+
+                            stages {
+                                stage("Deploy to Staging") {
+                                    steps {
+                                        gitlabCommitStatus(name: 'Deloy to Staging') {
+                                            container('milvus-testframework') {
+                                                script {
+                                                    load "${env.WORKSPACE}/ci/jenkinsfile/deploy2staging.groovy"
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                                stage("Staging Test") {
+                                    steps {
+                                        gitlabCommitStatus(name: 'Staging Test') {
+                                            container('milvus-testframework') {
+                                                script {
+                                                    load "${env.WORKSPACE}/ci/jenkinsfile/staging_test.groovy"
+                                                    load "${env.WORKSPACE}/ci/jenkinsfile/upload_staging_test_out.groovy"
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                                stage ("Cleanup Staging") {
+                                    steps {
+                                        gitlabCommitStatus(name: 'Cleanup Staging') {
+                                            container('milvus-testframework') {
+                                                script {
+                                                    load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            post {
+                                always {
+                                    container('milvus-testframework') {
+                                        script {
+                                            load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
+                                        }
+                                    }
+                                }
+                                success {
+                                    script {
+                                        echo "Milvus Deploy to Staging Single Node CI/CD success !"
+                                    }
+                                }
+                                aborted {
+                                    script {
+                                        echo "Milvus Deploy to Staging Single Node CI/CD aborted !"
+                                    }
+                                }
+                                failure {
+                                    script {
+                                        echo "Milvus Deploy to Staging Single Node CI/CD failure !"
                                     }
                                 }
                             }

cpp/CHANGELOG.md

@@ -5,6 +5,8 @@ Please mark all change in change log and use the ticket from JIRA.
 # Milvus 0.4.0 (2019-07-28)
 
 ## Bug
+- MS-411 - Fix metric unittest linking error
+- MS-412 - Fix gpu cache logical error
 
 ## Improvement
 - MS-327 - Clean code for milvus
@@ -46,6 +48,10 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-405 - Add delete task support
 - MS-407 - Reconstruct MetricsCollector
 - MS-408 - Add device_id in resource construct function
+- MS-409 - Using new scheduler
+- MS-413 - Remove thrift dependency
+- MS-410 - Add resource config comment
+- MS-419 - Move index_file_size from IndexParam to TableSchema
 
 ## New Feature
 - MS-343 - Implement ResourceMgr

cpp/CMakeLists.txt

@@ -34,6 +34,8 @@ else()
 endif()
 message(STATUS "Build type = ${BUILD_TYPE}")
 
+#add_definitions(-DNEW_SCHEDULER)
+
 project(milvus VERSION "${MILVUS_VERSION}")
 project(milvus_engine LANGUAGES CUDA CXX)
 
@@ -71,10 +73,6 @@ set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
 message("CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}")
 message("CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}")
 
-set(MILVUS_WITH_THRIFT "OFF")
-set(GPU_VERSION "ON")
-add_definitions("-DGPU_VERSION")
-
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED on)
 

cpp/cmake/DefineOptions.cmake

@@ -90,8 +90,6 @@ define_option(MILVUS_WITH_SQLITE_ORM "Build with SQLite ORM library" ON)
 
 define_option(MILVUS_WITH_MYSQLPP "Build with MySQL++" ON)
 
-define_option(MILVUS_WITH_THRIFT "Build with Apache Thrift library" OFF)
-
 define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
 
 define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)

cpp/cmake/ThirdPartyPackages.cmake

@@ -84,8 +84,6 @@ macro(build_dependency DEPENDENCY_NAME)
         build_sqlite()
     elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite_ORM")
         build_sqlite_orm()
-    elseif("${DEPENDENCY_NAME}" STREQUAL "Thrift")
-        build_thrift()
     elseif("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp")
         build_yamlcpp()
     elseif("${DEPENDENCY_NAME}" STREQUAL "ZLIB")
@@ -403,14 +401,6 @@ else()
 endif()
 set(SQLITE_ORM_MD5 "ba9a405a8a1421c093aa8ce988ff8598")
 
-if(DEFINED ENV{MILVUS_THRIFT_URL})
-    set(THRIFT_SOURCE_URL "$ENV{MILVUS_THRIFT_URL}")
-else()
-    set(THRIFT_SOURCE_URL
-            "https://github.com/apache/thrift/archive/${THRIFT_VERSION}.tar.gz")
-endif()
-set(THRIFT_MD5 "ff9af01fec424b5a279fa8a3c9e95c0c")
-
 if(DEFINED ENV{MILVUS_YAMLCPP_URL})
     set(YAMLCPP_SOURCE_URL "$ENV{MILVUS_YAMLCPP_URL}")
 else()
@@ -1943,127 +1933,6 @@ if(MILVUS_WITH_SQLITE_ORM)
     include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}")
 endif()
 
-# ----------------------------------------------------------------------
-# Thrift
-
-macro(build_thrift)
-    message(STATUS "Building Apache Thrift-${THRIFT_VERSION} from source")
-    set(THRIFT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix/src/thrift_ep")
-    set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include")
-    set(THRIFT_COMPILER "${THRIFT_PREFIX}/bin/thrift")
-    set(THRIFT_CMAKE_ARGS
-            ${EP_COMMON_CMAKE_ARGS}
-            "-DCMAKE_INSTALL_PREFIX=${THRIFT_PREFIX}"
-            "-DCMAKE_INSTALL_RPATH=${THRIFT_PREFIX}/lib"
-	    -DBOOST_ROOT=${BOOST_PREFIX}
-            -DWITH_CPP=ON
-            -DWITH_STATIC_LIB=ON
-	    -DBUILD_SHARED_LIBS=OFF
-	    -DBUILD_TESTING=OFF
-	    -DBUILD_EXAMPLES=OFF
-            -DBUILD_TUTORIALS=OFF
-            -DWITH_QT4=OFF
-            -DWITH_QT5=OFF
-            -DWITH_C_GLIB=OFF
-            -DWITH_JAVA=OFF
-            -DWITH_PYTHON=OFF
-            -DWITH_HASKELL=OFF
-            -DWITH_LIBEVENT=OFF
-            -DCMAKE_BUILD_TYPE=Release)
-
-    # Thrift also uses boost. Forward important boost settings if there were ones passed.
-    if(DEFINED BOOST_ROOT)
-        set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBOOST_ROOT=${BOOST_ROOT}")
-    endif()
-    if(DEFINED Boost_NAMESPACE)
-        set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBoost_NAMESPACE=${Boost_NAMESPACE}")
-    endif()
-
-    set(THRIFT_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}thrift")
-
-    set(THRIFT_STATIC_LIB
-            "${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
-
-    if(ZLIB_SHARED_LIB)
-        set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_SHARED_LIB}" ${THRIFT_CMAKE_ARGS})
-    else()
-        set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}" ${THRIFT_CMAKE_ARGS})
-    endif()
-    set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} ${ZLIB_LIBRARY})
-
-    if(USE_JFROG_CACHE STREQUAL "ON")
-        string(MD5 THRIFT_COMBINE_MD5 "${THRIFT_MD5}${ZLIB_MD5}")
-        set(THRIFT_CACHE_PACKAGE_NAME "thrift_${THRIFT_COMBINE_MD5}.tar.gz")
-        set(THRIFT_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${THRIFT_CACHE_PACKAGE_NAME}")
-        set(THRIFT_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${THRIFT_CACHE_PACKAGE_NAME}")
-
-        execute_process(COMMAND wget -q --method HEAD ${THRIFT_CACHE_URL} RESULT_VARIABLE return_code)
-        message(STATUS "Check the remote file ${THRIFT_CACHE_URL}. return code = ${return_code}")
-        if (NOT return_code EQUAL 0)
-            externalproject_add(thrift_ep
-                    URL
-                    ${THRIFT_SOURCE_URL}
-                    BUILD_BYPRODUCTS
-                    "${THRIFT_STATIC_LIB}"
-                    "${THRIFT_COMPILER}"
-                    BUILD_COMMAND
-                    ${MAKE}
-                    ${MAKE_BUILD_ARGS}
-                    CMAKE_ARGS
-                    ${THRIFT_CMAKE_ARGS}
-                    INSTALL_COMMAND
-                    ${MAKE} install
-                    DEPENDS
-                    ${THRIFT_DEPENDENCIES}
-                    ${EP_LOG_OPTIONS})
-
-            ExternalProject_Create_Cache(thrift_ep ${THRIFT_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${THRIFT_CACHE_URL})
-        else()
-            file(DOWNLOAD ${THRIFT_CACHE_URL} ${THRIFT_CACHE_PACKAGE_PATH} STATUS status)
-            list(GET status 0 status_code)
-            message(STATUS "DOWNLOADING FROM ${THRIFT_CACHE_URL} TO ${THRIFT_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-            if (status_code EQUAL 0)
-                ExternalProject_Use_Cache(thrift_ep ${THRIFT_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
-            endif()
-        endif()
-    else()
-        externalproject_add(thrift_ep
-                URL
-                ${THRIFT_SOURCE_URL}
-                BUILD_BYPRODUCTS
-                "${THRIFT_STATIC_LIB}"
-                "${THRIFT_COMPILER}"
-                BUILD_COMMAND
-                ${MAKE}
-                ${MAKE_BUILD_ARGS}
-                CMAKE_ARGS
-                ${THRIFT_CMAKE_ARGS}
-                INSTALL_COMMAND
-                ${MAKE} install
-                DEPENDS
-                ${THRIFT_DEPENDENCIES}
-                ${EP_LOG_OPTIONS})
-    endif()
-
-    # The include directory must exist before it is referenced by a target.
-    file(MAKE_DIRECTORY "${THRIFT_INCLUDE_DIR}")
-    add_library(thrift STATIC IMPORTED)
-    set_target_properties(thrift
-            PROPERTIES IMPORTED_LOCATION "${THRIFT_STATIC_LIB}"
-            INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}")
-    add_dependencies(thrift thrift_ep)
-endmacro()
-
-if(MILVUS_WITH_THRIFT)
-    resolve_dependency(Thrift)
-
-    link_directories(SYSTEM ${THRIFT_PREFIX}/lib/)
-    link_directories(SYSTEM ${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix/src/thrift_ep-build/lib)
-    include_directories(SYSTEM ${THRIFT_INCLUDE_DIR})
-    include_directories(SYSTEM ${THRIFT_PREFIX}/lib/cpp/src)
-    include_directories(SYSTEM ${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix/src/thrift_ep-build)
-endif()
-
 # ----------------------------------------------------------------------
 # yaml-cpp
 
@@ -2683,15 +2552,12 @@ macro(build_grpc)
     add_dependencies(grpc_protoc grpc_ep)
 endmacro()
 
-if(NOT MILVUS_WITH_THRIFT STREQUAL "ON")
-    resolve_dependency(GRPC)
+resolve_dependency(GRPC)
 
-    get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES)
-    include_directories(SYSTEM ${GRPC_INCLUDE_DIR})
-    link_directories(SYSTEM ${GRPC_PREFIX}/lib)
+get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES)
+include_directories(SYSTEM ${GRPC_INCLUDE_DIR})
+link_directories(SYSTEM ${GRPC_PREFIX}/lib)
 
-    set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party)
-    include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src)
-    link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR})
-
-endif()
+set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party)
+include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src)
+link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR})

cpp/conf/server_config.template

@@ -1,5 +1,5 @@
 server_config:
-  address: 0.0.0.0
+  address: 0.0.0.0            # milvus server ip address
   port: 19530                 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
   gpu_index: 0                # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1
   mode: single                # milvus deployment type: single, cluster, read_only
@@ -7,43 +7,83 @@ server_config:
 db_config:
   db_path: @MILVUS_DB_PATH@             # milvus data storage path
   db_slave_path:                        # secondry data storage path, split by semicolon
-
-  parallel_reduce: false                 # use multi-threads to reduce topk result
+  parallel_reduce: false                # use multi-threads to reduce topk result
 
   # URI format: dialect://username:password@host:port/database
   # All parts except dialect are optional, but you MUST include the delimiters
   # Currently dialect supports mysql or sqlite
   db_backend_url: sqlite://:@:/
 
-  index_building_threshold: 1024   # index building trigger threshold, default: 1024, unit: MB
   archive_disk_threshold: 0        # triger archive action if storage size exceed this value, 0 means no limit, unit: GB
   archive_days_threshold: 0        # files older than x days will be archived, 0 means no limit, unit: day
   insert_buffer_size: 4            # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
                                    # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
 
 metric_config:
-  is_startup: off      # if monitoring start: on, off
-  collector: prometheus # metrics collector: prometheus
-  prometheus_config:    # following are prometheus configure
-    port: 8080          # the port prometheus use to fetch metrics
-    push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address
-    push_gateway_port: 9091            # push method configure: push gateway port
+  is_startup: off                       # if monitoring start: on, off
+  collector: prometheus                 # metrics collector: prometheus
+  prometheus_config:                    # following are prometheus configure
+    port: 8080                          # the port prometheus use to fetch metrics
+    push_gateway_ip_address: 127.0.0.1  # push method configure: push gateway ip address
+    push_gateway_port: 9091             # push method configure: push gateway port
 
-
-license_config:         # license configure
-  license_path: "@MILVUS_DB_PATH@/system.license" # license file path
-
-cache_config:           # cache configure
-  cpu_cache_capacity: 16   # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
-  cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
-  insert_cache_immediately: false  # insert data will be load into cache immediately for hot query
-  gpu_cache_capacity: 5    # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory
-  gpu_cache_free_percent: 0.85     # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
-  gpu_ids: 0,1             # gpu id
+cache_config:
+  cpu_cache_capacity: 16            # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
+  cpu_cache_free_percent: 0.85      # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
+  insert_cache_immediately: false   # insert data will be load into cache immediately for hot query
+  gpu_cache_capacity: 5             # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory
+  gpu_cache_free_percent: 0.85      # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
+  gpu_ids: 0,1                      # gpu id
 
 engine_config:
-  nprobe: 10
-  nlist: 16384
   use_blas_threshold: 20
-  metric_type: L2               # compare vectors by euclidean distance(L2) or inner product(IP), optional: L2 or IP
   omp_thread_num: 0             # how many compute threads be used by engine, 0 means use all cpu core to compute
+
+resource_config:
+  # resource list, length: 0~N
+  # please set a DISK resource and a CPU resource least, or system will not return query result.
+  #
+  # example:
+  # resource_name:               # resource name, just using in connections below
+  #   type: DISK                 # resource type, optional: DISK/CPU/GPU
+  #   memory: 256                # memory size, unit: GB
+  #   device_id: 0
+  #   enable_loader: true        # if is enable loader, optional: true, false
+  #   enable_executor: false     # if is enable executor, optional: true, false
+
+  resources:
+    ssda:
+      type: DISK
+      memory: 2048
+      device_id: 0
+      enable_loader: true
+      enable_executor: false
+
+    cpu:
+      type: CPU
+      memory: 64
+      device_id: 0
+      enable_loader: true
+      enable_executor: false
+
+    gtx1060:
+      type: GPU
+      memory: 6
+      device_id: 0
+      enable_loader: true
+      enable_executor: true
+
+    gtx1660:
+      type: GPU
+      memory: 6
+      device_id: 1
+      enable_loader: true
+      enable_executor: true
+
+  # connection list, length: 0~N
+  # format: -${resource_name}===${resource_name}
+  connections:
+    - ssda===cpu
+    - cpu===gtx1060
+    - cpu===gtx1660
+

cpp/src/CMakeLists.txt

@@ -8,7 +8,6 @@ aux_source_directory(cache cache_files)
 aux_source_directory(config config_files)
 aux_source_directory(server server_files)
 aux_source_directory(server/grpc_impl grpcserver_files)
-aux_source_directory(server/thrift_impl thriftserver_files)
 aux_source_directory(utils utils_files)
 aux_source_directory(db db_main_files)
 aux_source_directory(db/engine db_engine_files)
@@ -17,6 +16,19 @@ aux_source_directory(db/meta db_meta_files)
 aux_source_directory(metrics metrics_files)
 aux_source_directory(wrapper/knowhere knowhere_files)
 
+aux_source_directory(scheduler/action scheduler_action_files)
+aux_source_directory(scheduler/event scheduler_event_files)
+aux_source_directory(scheduler/resource scheduler_resource_files)
+aux_source_directory(scheduler/task scheduler_task_files)
+aux_source_directory(scheduler scheduler_root_files)
+set(scheduler_srcs
+        ${scheduler_action_files}
+        ${scheduler_event_files}
+        ${scheduler_resource_files}
+        ${scheduler_task_files}
+        ${scheduler_root_files}
+        )
+
 aux_source_directory(db/scheduler scheduler_files)
 aux_source_directory(db/scheduler/context scheduler_context_files)
 aux_source_directory(db/scheduler/task scheduler_task_files)
@@ -35,17 +47,6 @@ set(license_generator_files
         license/LicenseLibrary.cpp
         )
 
-if (MILVUS_WITH_THRIFT STREQUAL "ON")
-set(thrift_service_files
-        thrift/gen-cpp/MilvusService.cpp
-        thrift/gen-cpp/milvus_constants.cpp
-        thrift/gen-cpp/milvus_types.cpp
-        metrics/SystemInfo.cpp
-        metrics/SystemInfo.h
-        server/thrift_impl/ThreadPoolServer.cpp
-        server/thrift_impl/ThreadPoolServer.h
-        )
-else()
 set(grpc_service_files
         metrics/SystemInfo.cpp
         metrics/SystemInfo.h
@@ -54,7 +55,6 @@ set(grpc_service_files
         grpc/gen-status/status.grpc.pb.cc
         grpc/gen-status/status.pb.cc
         )
-endif()
 
 set(db_files
         ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
@@ -74,25 +74,16 @@ set(s3_client_files
 
 include_directories(/usr/include)
 include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
-include_directories(thrift/gen-cpp)
 include_directories(/usr/include/mysql)
 include_directories(grpc/gen-status)
 include_directories(grpc/gen-milvus)
 
-if (MILVUS_WITH_THRIFT STREQUAL "ON")
-    message("Build thrift server")
-    add_definitions("-DMILVUS_ENABLE_THRIFT")
-    set(client_thrift_lib
-            thrift)
-else()
-    message("Build grpc server")
-    set(client_grpc_lib
-            grpcpp_channelz
-            grpc++
-            grpc
-            grpc_protobuf
-            grpc_protoc)
-endif()
+set(client_grpc_lib
+        grpcpp_channelz
+        grpc++
+        grpc
+        grpc_protobuf
+        grpc_protoc)
 
 set(third_party_libs
         knowhere
@@ -104,7 +95,6 @@ set(third_party_libs
         lapack
         easyloggingpp
         sqlite
-        ${client_thrift_lib}
         ${client_grpc_lib}
         yaml-cpp
         prometheus-cpp-push
@@ -125,9 +115,6 @@ set(third_party_libs
         cudart
         )
 
-if (MEGASEARCH_WITH_ARROW STREQUAL "ON")
-    set(third_party_libs ${third_party_libs} arrow)
-endif()
 if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
     set(third_party_libs ${third_party_libs}
             ${MKL_LIBS})
@@ -143,23 +130,14 @@ if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
             libunwind)
 endif()
 
-if (GPU_VERSION STREQUAL "ON")
-    link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
-    set(engine_libs
-            pthread
-            libgomp.a
-            libgfortran.a
-            ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
-            )
-else()
-    set(engine_libs
-            pthread
-            libgomp.a
-            libgfortran.a
-            ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
-            )
-endif ()
 
+link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
+set(engine_libs
+        pthread
+        libgomp.a
+        libgfortran.a
+        ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
+        )
 
 if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
     set(engine_libs
@@ -202,32 +180,17 @@ set(knowhere_libs
         tbb
         )
 
-if (MILVUS_WITH_THRIFT STREQUAL "ON")
-    add_executable(milvus_server
-            ${config_files}
-            ${server_files}
-            ${thriftserver_files}
-            ${utils_files}
-            ${thrift_service_files}
-            ${metrics_files}
-            )
-else()
-    add_executable(milvus_server
-            ${config_files}
-            ${server_files}
-            ${grpcserver_files}
-            ${utils_files}
-            ${grpc_service_files}
-            ${metrics_files}
-            )
-endif()
-
-if(MILVUS_WITH_THRIFT STREQUAL "ON")
-    target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_party_libs})
-else()
-    target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_party_libs})
-endif()
+add_executable(milvus_server
+        ${config_files}
+        ${server_files}
+        ${grpcserver_files}
+        ${utils_files}
+        ${grpc_service_files}
+        ${metrics_files}
+        ${scheduler_srcs}
+        )
 
+target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_party_libs})
 
 install(TARGETS milvus_server DESTINATION bin)
 

cpp/src/db/DBImpl.cpp

@@ -23,6 +23,7 @@
 #include <cstring>
 #include <cache/CpuCacheMgr.h>
 #include <boost/filesystem.hpp>
+#include "scheduler/SchedInst.h"
 #include <src/cache/GpuCacheMgr.h>
 
 namespace zilliz {
@@ -54,7 +55,9 @@ DBImpl::DBImpl(const Options& options)
 }
 
 Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
-    return meta_ptr_->CreateTable(table_schema);
+    meta::TableSchema temp_schema = table_schema;
+    temp_schema.index_file_size_ *= ONE_MB;
+    return meta_ptr_->CreateTable(temp_schema);
 }
 
 Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) {
@@ -67,13 +70,15 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date
 
         //scheduler will determine when to delete table files
         TaskScheduler& scheduler = TaskScheduler::GetInstance();
-        DeleteContextPtr context = std::make_shared<DeleteContext>(table_id, meta_ptr_);
+        DeleteContextPtr context = std::make_shared<DeleteContext>(table_id,
+                                                               meta_ptr_,
+                                                               ResMgrInst::GetInstance()->GetNumOfComputeResource());
         scheduler.Schedule(context);
+        context->WaitAndDelete();
     } else {
         meta_ptr_->DropPartitionsByDates(table_id, dates);
     }
 
-
     return Status::OK();
 }
 
@@ -397,7 +402,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
         ENGINE_LOG_DEBUG << "Merging file " << file_schema.file_id_;
         index_size = index->Size();
 
-        if (index_size >= options_.index_trigger_size) break;
+        if (index_size >= file_schema.index_file_size_) break;
     }
 
     //step 3: serialize to disk
@@ -547,6 +552,11 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
         //step 2: drop old index files
         DropIndex(table_id);
 
+        if(index.engine_type_ == (int)EngineType::FAISS_IDMAP) {
+            ENGINE_LOG_DEBUG << "index type = IDMAP, no need to build index";
+            return Status::OK();
+        }
+
         //step 3: update index info
 
         status = meta_ptr_->UpdateTableIndexParam(table_id, index);
@@ -608,7 +618,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
 
         try {
             server::CollectBuildIndexMetrics metrics;
-            index = to_index->BuildIndex(table_file.location_);
+            index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_);
         } catch (std::exception& ex) {
             //typical error: out of gpu memory
             std::string msg = "BuildIndex encounter exception" + std::string(ex.what());

cpp/src/db/Options.h

@@ -55,9 +55,8 @@ struct Options {
     } MODE;
 
     Options();
-    uint16_t  memory_sync_interval = 1;             //unit: second
+
     uint16_t  merge_trigger_number = 2;
-    size_t  index_trigger_size = ONE_GB;            //unit: byte
     DBMetaOptions meta;
     int mode = MODE::SINGLE;
 

cpp/src/db/Types.h

@@ -24,7 +24,6 @@ typedef std::vector<QueryResult> QueryResults;
 struct TableIndex {
     int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;
     int32_t nlist_ = 16384;
-    int32_t index_file_size_ = 1024; //MB
     int32_t metric_type_ = (int)MetricType::L2;
 };
 

cpp/src/db/Utils.cpp

@@ -149,7 +149,6 @@ Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
 bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) {
     return index1.engine_type_ == index2.engine_type_
         && index1.nlist_ == index2.nlist_
-        && index1.index_file_size_ == index2.index_file_size_
         && index1.metric_type_ == index2.metric_type_;
 }
 

cpp/src/db/engine/ExecutionEngine.h

@@ -51,6 +51,8 @@ public:
 
     virtual Status CopyToCpu() = 0;
 
+    virtual std::shared_ptr<ExecutionEngine> Clone() = 0;
+
     virtual Status Merge(const std::string& location) = 0;
 
     virtual Status Search(long n,
@@ -60,7 +62,7 @@ public:
                   float *distances,
                   long *labels) const = 0;
 
-    virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string&) = 0;
+    virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string& location, EngineType engine_type) = 0;
 
     virtual Status Cache() = 0;
 

cpp/src/db/engine/ExecutionEngineImpl.cpp

@@ -139,9 +139,11 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
 }
 
 Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
-    index_ = zilliz::milvus::cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_);
-    bool already_in_cache = (index_ != nullptr);
-    if (!index_) {
+    auto index = zilliz::milvus::cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_);
+    bool already_in_cache = (index != nullptr);
+    if (already_in_cache) {
+        index_ = index;
+    } else {
         try {
             index_ = index_->CopyToGpu(device_id);
             ENGINE_LOG_DEBUG << "CPU to GPU" << device_id;
@@ -161,9 +163,11 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
 }
 
 Status ExecutionEngineImpl::CopyToCpu() {
-    index_ = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
-    bool already_in_cache = (index_ != nullptr);
-    if (!index_) {
+    auto index = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
+    bool already_in_cache = (index != nullptr);
+    if (already_in_cache) {
+        index_ = index;
+    } else {
         try {
             index_ = index_->CopyToCpu();
             ENGINE_LOG_DEBUG << "GPU to CPU";
@@ -175,12 +179,19 @@ Status ExecutionEngineImpl::CopyToCpu() {
         }
     }
 
-    if(!already_in_cache) {
+    if (!already_in_cache) {
         Cache();
     }
     return Status::OK();
 }
 
+ExecutionEnginePtr ExecutionEngineImpl::Clone() {
+    auto ret = std::make_shared<ExecutionEngineImpl>(dim_, location_, index_type_, metric_type_, nlist_);
+    ret->Init();
+    ret->index_ = index_->Clone();
+    return ret;
+}
+
 Status ExecutionEngineImpl::Merge(const std::string &location) {
     if (location == location_) {
         return Status::Error("Cannot Merge Self");
@@ -214,11 +225,11 @@ Status ExecutionEngineImpl::Merge(const std::string &location) {
 }
 
 ExecutionEnginePtr
-ExecutionEngineImpl::BuildIndex(const std::string &location) {
+ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_type) {
     ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_;
 
     auto from_index = std::dynamic_pointer_cast<BFIndex>(index_);
-    auto to_index = CreatetVecIndex(index_type_);
+    auto to_index = CreatetVecIndex(engine_type);
     if (!to_index) {
         throw Exception("Create Empty VecIndex");
     }
@@ -236,7 +247,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location) {
                                  build_cfg);
     if (ec != server::KNOWHERE_SUCCESS) { throw Exception("Build index error"); }
 
-    return std::make_shared<ExecutionEngineImpl>(to_index, location, index_type_, metric_type_, nlist_);
+    return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
 }
 
 Status ExecutionEngineImpl::Search(long n,
@@ -269,7 +280,7 @@ Status ExecutionEngineImpl::Init() {
     using namespace zilliz::milvus::server;
     ServerConfig &config = ServerConfig::GetInstance();
     ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
-        gpu_num_ = server_config.GetInt32Value("gpu_index", 0);
+    gpu_num_ = server_config.GetInt32Value("gpu_index", 0);
 
     return Status::OK();
 }

cpp/src/db/engine/ExecutionEngineImpl.h

@@ -50,6 +50,8 @@ public:
 
     Status CopyToCpu() override;
 
+    ExecutionEnginePtr Clone() override;
+
     Status Merge(const std::string &location) override;
 
     Status Search(long n,
@@ -59,7 +61,7 @@ public:
                   float *distances,
                   long *labels) const override;
 
-    ExecutionEnginePtr BuildIndex(const std::string &) override;
+    ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override;
 
     Status Cache() override;
 

cpp/src/db/insert/MemTableFile.cpp

@@ -87,7 +87,7 @@ Status MemTableFile::Serialize() {
     table_file_schema_.file_size_ = execution_engine_->PhysicalSize();
     table_file_schema_.row_count_ = execution_engine_->Count();
 
-    table_file_schema_.file_type_ = (size >= options_.index_trigger_size) ?
+    table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ?
                                     meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
 
     auto status = meta_->UpdateTableFile(table_file_schema_);

cpp/src/db/meta/MetaTypes.h

@@ -19,7 +19,7 @@ namespace meta {
 
 constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
 constexpr int32_t DEFAULT_NLIST = 16384;
-constexpr int32_t DEFAULT_INDEX_FILE_SIZE = 1024*ONE_MB;
+constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
 constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
 
 constexpr int64_t FLAG_MASK_USERID = 1;
@@ -40,9 +40,9 @@ struct TableSchema {
     uint16_t dimension_ = 0;
     int64_t created_on_ = 0;
     int64_t flag_ = 0;
+    int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE;
     int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
     int32_t nlist_ = DEFAULT_NLIST;
-    int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE;
     int32_t metric_type_ = DEFAULT_METRIC_TYPE;
 }; // TableSchema
 
@@ -69,6 +69,7 @@ struct TableFileSchema {
     std::string location_;
     int64_t updated_time_ = 0;
     int64_t created_on_ = 0;
+    int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta
     int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
     int32_t nlist_ = DEFAULT_NLIST; //not persist to meta
     int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta

cpp/src/db/meta/MySQLMetaImpl.cpp

@@ -138,9 +138,9 @@ Status MySQLMetaImpl::Initialize() {
                                 "dimension SMALLINT NOT NULL, " <<
                                 "created_on BIGINT NOT NULL, " <<
                                 "flag BIGINT DEFAULT 0 NOT NULL, " <<
+                                "index_file_size INT DEFAULT 1024 NOT NULL, " <<
                                 "engine_type INT DEFAULT 1 NOT NULL, " <<
                                 "nlist INT DEFAULT 16384 NOT NULL, " <<
-                                "index_file_size INT DEFAULT 1024 NOT NULL, " <<
                                 "metric_type INT DEFAULT 1 NOT NULL);";
 
                 ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
@@ -407,7 +407,6 @@ Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const T
                                            "created_on = " << created_on << ", " <<
                                            "engine_type_ = " << index.engine_type_ << ", " <<
                                            "nlist = " << index.nlist_ << ", " <<
-                                           "index_file_size = " << index.index_file_size_*ONE_MB << ", " <<
                                            "metric_type = " << index.metric_type_ << " " <<
                                            "WHERE id = " << quote << table_id << ";";
 
@@ -504,7 +503,6 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex
 
                 index.engine_type_ = resRow["engine_type"];
                 index.nlist_ = resRow["nlist"];
-                index.index_file_size_ = resRow["index_file_size"]/ONE_MB;
                 index.metric_type_ = resRow["metric_type"];
             } else {
                 return Status::NotFound("Table " + table_id + " not found");
@@ -694,12 +692,12 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) {
 
             table_schema.dimension_ = resRow["dimension"];
 
+            table_schema.index_file_size_ = resRow["index_file_size"];
+
             table_schema.engine_type_ = resRow["engine_type"];
 
             table_schema.nlist_ = resRow["nlist"];
 
-            table_schema.index_file_size_ = resRow["index_file_size"];
-
             table_schema.metric_type_ = resRow["metric_type"];
         } else {
             return Status::NotFound("Table " + table_schema.table_id_ + " not found");
@@ -790,12 +788,12 @@ Status MySQLMetaImpl::AllTables(std::vector<TableSchema> &table_schema_array) {
 
             table_schema.dimension_ = resRow["dimension"];
 
+            table_schema.index_file_size_ = resRow["index_file_size"];
+
             table_schema.engine_type_ = resRow["engine_type"];
 
             table_schema.nlist_ = resRow["nlist"];
 
-            table_schema.index_file_size_ = resRow["index_file_size"];
-
             table_schema.metric_type_ = resRow["metric_type"];
 
             table_schema_array.emplace_back(table_schema);
@@ -833,6 +831,7 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
         file_schema.row_count_ = 0;
         file_schema.created_on_ = utils::GetMicroSecTimeStamp();
         file_schema.updated_time_ = file_schema.created_on_;
+        file_schema.index_file_size_ = table_schema.index_file_size_;
         file_schema.engine_type_ = table_schema.engine_type_;
         file_schema.nlist_ = table_schema.nlist_;
         file_schema.metric_type_ = table_schema.metric_type_;
@@ -949,9 +948,10 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) {
                 groups[table_file.table_id_] = table_schema;
 
             }
-            table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
-            table_file.nlist_ = groups[table_file.table_id_].nlist_;
             table_file.dimension_ = groups[table_file.table_id_].dimension_;
+            table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_;
+            table_file.nlist_ = groups[table_file.table_id_].nlist_;
+            table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
 
             utils::GetTableFilePath(options_, table_file);
 
@@ -1041,12 +1041,14 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
             resRow["table_id"].to_string(table_id_str);
             table_file.table_id_ = table_id_str;
 
+            table_file.index_file_size_ = table_schema.index_file_size_;
+
             table_file.engine_type_ = resRow["engine_type"];
 
-            table_file.metric_type_ = table_schema.metric_type_;
-
             table_file.nlist_ = table_schema.nlist_;
 
+            table_file.metric_type_ = table_schema.metric_type_;
+
             std::string file_id;
             resRow["file_id"].to_string(file_id);
             table_file.file_id_ = file_id;
@@ -1153,12 +1155,14 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
             resRow["table_id"].to_string(table_id_str);
             table_file.table_id_ = table_id_str;
 
+            table_file.index_file_size_ = table_schema.index_file_size_;
+
             table_file.engine_type_ = resRow["engine_type"];
 
-            table_file.metric_type_ = table_schema.metric_type_;
-
             table_file.nlist_ = table_schema.nlist_;
 
+            table_file.metric_type_ = table_schema.metric_type_;
+
             std::string file_id;
             resRow["file_id"].to_string(file_id);
             table_file.file_id_ = file_id;
@@ -1253,12 +1257,14 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
 
             table_file.date_ = resRow["date"];
 
+            table_file.index_file_size_ = table_schema.index_file_size_;
+
             table_file.engine_type_ = resRow["engine_type"];
 
-            table_file.metric_type_ = table_schema.metric_type_;
-
             table_file.nlist_ = table_schema.nlist_;
 
+            table_file.metric_type_ = table_schema.metric_type_;
+
             table_file.created_on_ = resRow["created_on"];
 
             table_file.dimension_ = table_schema.dimension_;
@@ -1336,12 +1342,14 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
 
             file_schema.table_id_ = table_id;
 
+            file_schema.index_file_size_ = table_schema.index_file_size_;
+
             file_schema.engine_type_ = resRow["engine_type"];
 
-            file_schema.metric_type_ = table_schema.metric_type_;
-
             file_schema.nlist_ = table_schema.nlist_;
 
+            file_schema.metric_type_ = table_schema.metric_type_;
+
             std::string file_id;
             resRow["file_id"].to_string(file_id);
             file_schema.file_id_ = file_id;

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -45,9 +45,9 @@ inline auto StoragePrototype(const std::string &path) {
                                    make_column("dimension", &TableSchema::dimension_),
                                    make_column("created_on", &TableSchema::created_on_),
                                    make_column("flag", &TableSchema::flag_, default_value(0)),
+                                   make_column("index_file_size", &TableSchema::index_file_size_),
                                    make_column("engine_type", &TableSchema::engine_type_),
                                    make_column("nlist", &TableSchema::nlist_),
-                                   make_column("index_file_size", &TableSchema::index_file_size_),
                                    make_column("metric_type", &TableSchema::metric_type_)),
                         make_table("TableFiles",
                                    make_column("id", &TableFileSchema::id_, primary_key()),
@@ -251,9 +251,9 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) {
                                                    &TableSchema::dimension_,
                                                    &TableSchema::created_on_,
                                                    &TableSchema::flag_,
+                                                   &TableSchema::index_file_size_,
                                                    &TableSchema::engine_type_,
                                                    &TableSchema::nlist_,
-                                                   &TableSchema::index_file_size_,
                                                    &TableSchema::metric_type_),
                                            where(c(&TableSchema::table_id_) == table_schema.table_id_
                                                  and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE));
@@ -264,9 +264,9 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) {
             table_schema.dimension_ = std::get<2>(groups[0]);
             table_schema.created_on_ = std::get<3>(groups[0]);
             table_schema.flag_ = std::get<4>(groups[0]);
-            table_schema.engine_type_ = std::get<5>(groups[0]);
-            table_schema.nlist_ = std::get<6>(groups[0]);
-            table_schema.index_file_size_ = std::get<7>(groups[0]);
+            table_schema.index_file_size_ = std::get<5>(groups[0]);
+            table_schema.engine_type_ = std::get<6>(groups[0]);
+            table_schema.nlist_ = std::get<7>(groups[0]);
             table_schema.metric_type_ = std::get<8>(groups[0]);
         } else {
             return Status::NotFound("Table " + table_schema.table_id_ + " not found");
@@ -344,7 +344,8 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const
                                                    &TableSchema::state_,
                                                    &TableSchema::dimension_,
                                                    &TableSchema::created_on_,
-                                                   &TableSchema::flag_),
+                                                   &TableSchema::flag_,
+                                                   &TableSchema::index_file_size_),
                                            where(c(&TableSchema::table_id_) == table_id
                                                  and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE));
 
@@ -356,9 +357,9 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const
             table_schema.dimension_ = std::get<2>(tables[0]);
             table_schema.created_on_ = std::get<3>(tables[0]);
             table_schema.flag_ = std::get<4>(tables[0]);
+            table_schema.index_file_size_ = std::get<5>(tables[0]);
             table_schema.engine_type_ = index.engine_type_;
             table_schema.nlist_ = index.nlist_;
-            table_schema.index_file_size_ = index.index_file_size_*ONE_MB;
             table_schema.metric_type_ = index.metric_type_;
 
             ConnectorPtr->update(table_schema);
@@ -420,8 +421,7 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde
         if (groups.size() == 1) {
             index.engine_type_ = std::get<0>(groups[0]);
             index.nlist_ = std::get<1>(groups[0]);
-            index.index_file_size_ = std::get<2>(groups[0])/ONE_MB;
-            index.metric_type_ = std::get<3>(groups[0]);
+            index.metric_type_ = std::get<2>(groups[0]);
         } else {
             return Status::NotFound("Table " + table_id + " not found");
         }
@@ -499,9 +499,9 @@ Status SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
                                                      &TableSchema::dimension_,
                                                      &TableSchema::created_on_,
                                                      &TableSchema::flag_,
+                                                     &TableSchema::index_file_size_,
                                                      &TableSchema::engine_type_,
                                                      &TableSchema::nlist_,
-                                                     &TableSchema::index_file_size_,
                                                      &TableSchema::metric_type_),
                                              where(c(&TableSchema::state_) != (int)TableSchema::TO_DELETE));
         for (auto &table : selected) {
@@ -511,9 +511,9 @@ Status SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
             schema.dimension_ = std::get<2>(table);
             schema.created_on_ = std::get<3>(table);
             schema.flag_ = std::get<4>(table);
-            schema.engine_type_ = std::get<5>(table);
-            schema.nlist_ = std::get<6>(table);
-            schema.index_file_size_ = std::get<7>(table);
+            schema.index_file_size_ = std::get<5>(table);
+            schema.engine_type_ = std::get<6>(table);
+            schema.nlist_ = std::get<7>(table);
             schema.metric_type_ = std::get<8>(table);
 
             table_schema_array.emplace_back(schema);
@@ -546,6 +546,7 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
         file_schema.row_count_ = 0;
         file_schema.created_on_ = utils::GetMicroSecTimeStamp();
         file_schema.updated_time_ = file_schema.created_on_;
+        file_schema.index_file_size_ = table_schema.index_file_size_;
         file_schema.engine_type_ = table_schema.engine_type_;
         file_schema.nlist_ = table_schema.nlist_;
         file_schema.metric_type_ = table_schema.metric_type_;
@@ -608,9 +609,10 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
                 }
                 groups[table_file.table_id_] = table_schema;
             }
-            table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
-            table_file.nlist_ = groups[table_file.table_id_].nlist_;
             table_file.dimension_ = groups[table_file.table_id_].dimension_;
+            table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_;
+            table_file.nlist_ = groups[table_file.table_id_].nlist_;
+            table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
             files.push_back(table_file);
         }
 
@@ -660,9 +662,11 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
                 table_file.row_count_ = std::get<5>(file);
                 table_file.date_ = std::get<6>(file);
                 table_file.engine_type_ = std::get<7>(file);
-                table_file.metric_type_ = table_schema.metric_type_;
-                table_file.nlist_ = table_schema.nlist_;
                 table_file.dimension_ = table_schema.dimension_;
+                table_file.index_file_size_ = table_schema.index_file_size_;
+                table_file.nlist_ = table_schema.nlist_;
+                table_file.metric_type_ = table_schema.metric_type_;
+
                 utils::GetTableFilePath(options_, table_file);
                 auto dateItr = files.find(table_file.date_);
                 if (dateItr == files.end()) {
@@ -703,9 +707,11 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
                 table_file.row_count_ = std::get<5>(file);
                 table_file.date_ = std::get<6>(file);
                 table_file.engine_type_ = std::get<7>(file);
-                table_file.metric_type_ = table_schema.metric_type_;
-                table_file.nlist_ = table_schema.nlist_;
                 table_file.dimension_ = table_schema.dimension_;
+                table_file.index_file_size_ = table_schema.index_file_size_;
+                table_file.nlist_ = table_schema.nlist_;
+                table_file.metric_type_ = table_schema.metric_type_;
+
                 utils::GetTableFilePath(options_, table_file);
                 auto dateItr = files.find(table_file.date_);
                 if (dateItr == files.end()) {
@@ -782,8 +788,10 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
             table_file.date_ = std::get<6>(file);
             table_file.engine_type_ = std::get<7>(file);
             table_file.dimension_ = table_schema.dimension_;
-            table_file.metric_type_ = table_schema.metric_type_;
+            table_file.index_file_size_ = table_schema.index_file_size_;
             table_file.nlist_ = table_schema.nlist_;
+            table_file.metric_type_ = table_schema.metric_type_;
+
             utils::GetTableFilePath(options_, table_file);
             auto dateItr = files.find(table_file.date_);
             if (dateItr == files.end()) {
@@ -842,8 +850,10 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
             table_file.date_ = std::get<6>(file);
             table_file.created_on_ = std::get<7>(file);
             table_file.dimension_ = table_schema.dimension_;
-            table_file.metric_type_ = table_schema.metric_type_;
+            table_file.index_file_size_ = table_schema.index_file_size_;
             table_file.nlist_ = table_schema.nlist_;
+            table_file.metric_type_ = table_schema.metric_type_;
+
             utils::GetTableFilePath(options_, table_file);
             auto dateItr = files.find(table_file.date_);
             if (dateItr == files.end()) {
@@ -892,10 +902,11 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
             file_schema.row_count_ = std::get<4>(file);
             file_schema.date_ = std::get<5>(file);
             file_schema.engine_type_ = std::get<6>(file);
-            file_schema.metric_type_ = table_schema.metric_type_;
-            file_schema.nlist_ = table_schema.nlist_;
             file_schema.created_on_ = std::get<7>(file);
             file_schema.dimension_ = table_schema.dimension_;
+            file_schema.index_file_size_ = table_schema.index_file_size_;
+            file_schema.nlist_ = table_schema.nlist_;
+            file_schema.metric_type_ = table_schema.metric_type_;
 
             utils::GetTableFilePath(options_, file_schema);
 

cpp/src/db/scheduler/TaskScheduler.cpp

@@ -4,12 +4,15 @@
  * Proprietary and confidential.
  ******************************************************************************/
 
+#include "server/ServerConfig.h"
 #include "TaskScheduler.h"
 #include "TaskDispatchQueue.h"
 #include "utils/Log.h"
 #include "utils/TimeRecorder.h"
 #include "db/engine/EngineFactory.h"
 #include "scheduler/task/TaskConvert.h"
+#include "scheduler/SchedInst.h"
+#include "scheduler/ResourceFactory.h"
 
 namespace zilliz {
 namespace milvus {
@@ -86,14 +89,22 @@ TaskScheduler::TaskDispatchWorker() {
             return true;
         }
 
+#ifdef NEW_SCHEDULER
         // TODO: Put task into Disk-TaskTable
-//        auto task = TaskConvert(task_ptr);
-//        DiskResourcePtr->task_table().Put(task)
+        auto task = TaskConvert(task_ptr);
+        auto disk_list = ResMgrInst::GetInstance()->GetDiskResources();
+        if (!disk_list.empty()) {
+	    if (auto disk = disk_list[0].lock()) {
+                disk->task_table().Put(task);
+            }
+        }
+#else
         //execute task
         ScheduleTaskPtr next_task = task_ptr->Execute();
         if(next_task != nullptr) {
             task_queue_.Put(next_task);
         }
+#endif
     }
 
     return true;

cpp/src/db/scheduler/context/DeleteContext.cpp

@@ -6,17 +6,35 @@
 
 #include "DeleteContext.h"
 
+
 namespace zilliz {
 namespace milvus {
 namespace engine {
 
-DeleteContext::DeleteContext(const std::string& table_id, meta::Meta::Ptr& meta_ptr)
+DeleteContext::DeleteContext(const std::string &table_id, meta::Meta::Ptr &meta_ptr, uint64_t num_resource)
     : IScheduleContext(ScheduleContextType::kDelete),
       table_id_(table_id),
-      meta_ptr_(meta_ptr) {
+      meta_ptr_(meta_ptr),
+      num_resource_(num_resource) {
 
 }
 
+void DeleteContext::WaitAndDelete() {
+#ifdef NEW_SCHEDULER
+    std::unique_lock<std::mutex> lock(mutex_);
+    cv_.wait(lock, [&] { return done_resource == num_resource_; });
+    meta_ptr_->DeleteTableFiles(table_id_);
+#endif
+}
+
+void DeleteContext::ResourceDone() {
+    {
+        std::lock_guard<std::mutex> lock(mutex_);
+        ++done_resource;
+    }
+    cv_.notify_one();
+}
+
 }
 }
 }

cpp/src/db/scheduler/context/DeleteContext.h

@@ -7,6 +7,8 @@
 
 #include "IScheduleContext.h"
 #include "db/meta/Meta.h"
+#include <mutex>
+#include <condition_variable>
 
 namespace zilliz {
 namespace milvus {
@@ -14,14 +16,21 @@ namespace engine {
 
 class DeleteContext : public IScheduleContext {
 public:
-    DeleteContext(const std::string& table_id, meta::Meta::Ptr& meta_ptr);
+    DeleteContext(const std::string& table_id, meta::Meta::Ptr& meta_ptr, uint64_t num_resource);
 
     std::string table_id() const { return table_id_; }
     meta::Meta::Ptr meta() const { return meta_ptr_; }
+    void WaitAndDelete();
+    void ResourceDone();
 
 private:
     std::string table_id_;
     meta::Meta::Ptr meta_ptr_;
+
+    uint64_t num_resource_;
+    uint64_t done_resource = 0;
+    std::mutex mutex_;
+    std::condition_variable cv_;
 };
 
 using DeleteContextPtr = std::shared_ptr<DeleteContext>;

cpp/src/db/scheduler/task/DeleteTask.h

@@ -18,7 +18,7 @@ public:
 
     virtual std::shared_ptr<IScheduleTask> Execute() override;
 
-private:
+public:
     DeleteContextPtr context_;
 };
 

cpp/src/grpc/gen-milvus/milvus.pb.cc

@@ -366,6 +366,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT
   ~0u,  // no _weak_field_map_
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, table_name_),
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, dimension_),
+  PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, index_file_size_),
   ~0u,  // no _has_bits_
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::Range, _internal_metadata_),
   ~0u,  // no _extensions_
@@ -459,7 +460,6 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT
   ~0u,  // no _weak_field_map_
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_type_),
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, nlist_),
-  PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_file_size_),
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, metric_type_),
   ~0u,  // no _has_bits_
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, _internal_metadata_),
@@ -479,19 +479,19 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT
 static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
   { 0, -1, sizeof(::milvus::grpc::TableName)},
   { 7, -1, sizeof(::milvus::grpc::TableSchema)},
-  { 14, -1, sizeof(::milvus::grpc::Range)},
-  { 21, -1, sizeof(::milvus::grpc::RowRecord)},
-  { 27, -1, sizeof(::milvus::grpc::InsertParam)},
-  { 35, -1, sizeof(::milvus::grpc::VectorIds)},
-  { 42, -1, sizeof(::milvus::grpc::SearchParam)},
-  { 52, -1, sizeof(::milvus::grpc::SearchInFilesParam)},
-  { 59, -1, sizeof(::milvus::grpc::QueryResult)},
-  { 66, -1, sizeof(::milvus::grpc::TopKQueryResult)},
-  { 73, -1, sizeof(::milvus::grpc::StringReply)},
-  { 80, -1, sizeof(::milvus::grpc::BoolReply)},
-  { 87, -1, sizeof(::milvus::grpc::TableRowCount)},
-  { 94, -1, sizeof(::milvus::grpc::Command)},
-  { 100, -1, sizeof(::milvus::grpc::Index)},
+  { 15, -1, sizeof(::milvus::grpc::Range)},
+  { 22, -1, sizeof(::milvus::grpc::RowRecord)},
+  { 28, -1, sizeof(::milvus::grpc::InsertParam)},
+  { 36, -1, sizeof(::milvus::grpc::VectorIds)},
+  { 43, -1, sizeof(::milvus::grpc::SearchParam)},
+  { 53, -1, sizeof(::milvus::grpc::SearchInFilesParam)},
+  { 60, -1, sizeof(::milvus::grpc::QueryResult)},
+  { 67, -1, sizeof(::milvus::grpc::TopKQueryResult)},
+  { 74, -1, sizeof(::milvus::grpc::StringReply)},
+  { 81, -1, sizeof(::milvus::grpc::BoolReply)},
+  { 88, -1, sizeof(::milvus::grpc::TableRowCount)},
+  { 95, -1, sizeof(::milvus::grpc::Command)},
+  { 101, -1, sizeof(::milvus::grpc::Index)},
   { 109, -1, sizeof(::milvus::grpc::IndexParam)},
   { 116, -1, sizeof(::milvus::grpc::DeleteByRangeParam)},
 };
@@ -519,35 +519,35 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] =
 const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) =
   "\n\014milvus.proto\022\013milvus.grpc\032\014status.prot"
   "o\"D\n\tTableName\022#\n\006status\030\001 \001(\0132\023.milvus."
-  "grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"L\n\013Table"
+  "grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"e\n\013Table"
   "Schema\022*\n\ntable_name\030\001 \001(\0132\026.milvus.grpc"
-  ".TableName\022\021\n\tdimension\030\002 \001(\003\"/\n\005Range\022\023"
-  "\n\013start_value\030\001 \001(\t\022\021\n\tend_value\030\002 \001(\t\" "
-  "\n\tRowRecord\022\023\n\013vector_data\030\001 \003(\002\"i\n\013Inse"
-  "rtParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row_recor"
-  "d_array\030\002 \003(\0132\026.milvus.grpc.RowRecord\022\024\n"
-  "\014row_id_array\030\003 \003(\003\"I\n\tVectorIds\022#\n\006stat"
-  "us\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017vector_"
-  "id_array\030\002 \003(\003\"\242\001\n\013SearchParam\022\022\n\ntable_"
-  "name\030\001 \001(\t\0222\n\022query_record_array\030\002 \003(\0132\026"
-  ".milvus.grpc.RowRecord\022-\n\021query_range_ar"
-  "ray\030\003 \003(\0132\022.milvus.grpc.Range\022\014\n\004topk\030\004 "
-  "\001(\003\022\016\n\006nprobe\030\005 \001(\003\"[\n\022SearchInFilesPara"
-  "m\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014search_param"
-  "\030\002 \001(\0132\030.milvus.grpc.SearchParam\"+\n\013Quer"
-  "yResult\022\n\n\002id\030\001 \001(\003\022\020\n\010distance\030\002 \001(\001\"m\n"
-  "\017TopKQueryResult\022#\n\006status\030\001 \001(\0132\023.milvu"
-  "s.grpc.Status\0225\n\023query_result_arrays\030\002 \003"
-  "(\0132\030.milvus.grpc.QueryResult\"H\n\013StringRe"
-  "ply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status"
-  "\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolReply\022#\n\006s"
-  "tatus\030\001 \001(\0132\023.milvus.grpc.Status\022\022\n\nbool"
-  "_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006status\030"
-  "\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017table_row_"
-  "count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(\t\"X\n\005I"
-  "ndex\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\022"
-  "\027\n\017index_file_size\030\003 \001(\005\022\023\n\013metric_type\030"
-  "\004 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132"
+  ".TableName\022\021\n\tdimension\030\002 \001(\003\022\027\n\017index_f"
+  "ile_size\030\003 \001(\003\"/\n\005Range\022\023\n\013start_value\030\001"
+  " \001(\t\022\021\n\tend_value\030\002 \001(\t\" \n\tRowRecord\022\023\n\013"
+  "vector_data\030\001 \003(\002\"i\n\013InsertParam\022\022\n\ntabl"
+  "e_name\030\001 \001(\t\0220\n\020row_record_array\030\002 \003(\0132\026"
+  ".milvus.grpc.RowRecord\022\024\n\014row_id_array\030\003"
+  " \003(\003\"I\n\tVectorIds\022#\n\006status\030\001 \001(\0132\023.milv"
+  "us.grpc.Status\022\027\n\017vector_id_array\030\002 \003(\003\""
+  "\242\001\n\013SearchParam\022\022\n\ntable_name\030\001 \001(\t\0222\n\022q"
+  "uery_record_array\030\002 \003(\0132\026.milvus.grpc.Ro"
+  "wRecord\022-\n\021query_range_array\030\003 \003(\0132\022.mil"
+  "vus.grpc.Range\022\014\n\004topk\030\004 \001(\003\022\016\n\006nprobe\030\005"
+  " \001(\003\"[\n\022SearchInFilesParam\022\025\n\rfile_id_ar"
+  "ray\030\001 \003(\t\022.\n\014search_param\030\002 \001(\0132\030.milvus"
+  ".grpc.SearchParam\"+\n\013QueryResult\022\n\n\002id\030\001"
+  " \001(\003\022\020\n\010distance\030\002 \001(\001\"m\n\017TopKQueryResul"
+  "t\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\0225"
+  "\n\023query_result_arrays\030\002 \003(\0132\030.milvus.grp"
+  "c.QueryResult\"H\n\013StringReply\022#\n\006status\030\001"
+  " \001(\0132\023.milvus.grpc.Status\022\024\n\014string_repl"
+  "y\030\002 \001(\t\"D\n\tBoolReply\022#\n\006status\030\001 \001(\0132\023.m"
+  "ilvus.grpc.Status\022\022\n\nbool_reply\030\002 \001(\010\"M\n"
+  "\rTableRowCount\022#\n\006status\030\001 \001(\0132\023.milvus."
+  "grpc.Status\022\027\n\017table_row_count\030\002 \001(\003\"\026\n\007"
+  "Command\022\013\n\003cmd\030\001 \001(\t\"\?\n\005Index\022\022\n\nindex_t"
+  "ype\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\022\023\n\013metric_type\030"
+  "\003 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132"
   "\026.milvus.grpc.TableName\022!\n\005index\030\002 \001(\0132\022"
   ".milvus.grpc.Index\"K\n\022DeleteByRangeParam"
   "\022!\n\005range\030\001 \001(\0132\022.milvus.grpc.Range\022\022\n\nt"
@@ -978,15 +978,17 @@ TableSchema::TableSchema(const TableSchema& from)
   } else {
     table_name_ = nullptr;
   }
-  dimension_ = from.dimension_;
+  ::memcpy(&dimension_, &from.dimension_,
+    static_cast<size_t>(reinterpret_cast<char*>(&index_file_size_) -
+    reinterpret_cast<char*>(&dimension_)) + sizeof(index_file_size_));
   // @@protoc_insertion_point(copy_constructor:milvus.grpc.TableSchema)
 }
 
 void TableSchema::SharedCtor() {
   ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_TableSchema_milvus_2eproto.base);
   ::memset(&table_name_, 0, static_cast<size_t>(
-      reinterpret_cast<char*>(&dimension_) -
-      reinterpret_cast<char*>(&table_name_)) + sizeof(dimension_));
+      reinterpret_cast<char*>(&index_file_size_) -
+      reinterpret_cast<char*>(&table_name_)) + sizeof(index_file_size_));
 }
 
 TableSchema::~TableSchema() {
@@ -1017,7 +1019,9 @@ void TableSchema::Clear() {
     delete table_name_;
   }
   table_name_ = nullptr;
-  dimension_ = PROTOBUF_LONGLONG(0);
+  ::memset(&dimension_, 0, static_cast<size_t>(
+      reinterpret_cast<char*>(&index_file_size_) -
+      reinterpret_cast<char*>(&dimension_)) + sizeof(index_file_size_));
   _internal_metadata_.Clear();
 }
 
@@ -1043,6 +1047,13 @@ const char* TableSchema::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID
           CHK_(ptr);
         } else goto handle_unusual;
         continue;
+      // int64 index_file_size = 3;
+      case 3:
+        if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) {
+          index_file_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
+          CHK_(ptr);
+        } else goto handle_unusual;
+        continue;
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
@@ -1097,6 +1108,19 @@ bool TableSchema::MergePartialFromCodedStream(
         break;
       }
 
+      // int64 index_file_size = 3;
+      case 3: {
+        if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) {
+
+          DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
+                   ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>(
+                 input, &index_file_size_)));
+        } else {
+          goto handle_unusual;
+        }
+        break;
+      }
+
       default: {
       handle_unusual:
         if (tag == 0) {
@@ -1135,6 +1159,11 @@ void TableSchema::SerializeWithCachedSizes(
     ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(2, this->dimension(), output);
   }
 
+  // int64 index_file_size = 3;
+  if (this->index_file_size() != 0) {
+    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->index_file_size(), output);
+  }
+
   if (_internal_metadata_.have_unknown_fields()) {
     ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
         _internal_metadata_.unknown_fields(), output);
@@ -1160,6 +1189,11 @@ void TableSchema::SerializeWithCachedSizes(
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(2, this->dimension(), target);
   }
 
+  // int64 index_file_size = 3;
+  if (this->index_file_size() != 0) {
+    target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->index_file_size(), target);
+  }
+
   if (_internal_metadata_.have_unknown_fields()) {
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
         _internal_metadata_.unknown_fields(), target);
@@ -1195,6 +1229,13 @@ size_t TableSchema::ByteSizeLong() const {
         this->dimension());
   }
 
+  // int64 index_file_size = 3;
+  if (this->index_file_size() != 0) {
+    total_size += 1 +
+      ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size(
+        this->index_file_size());
+  }
+
   int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
   SetCachedSize(cached_size);
   return total_size;
@@ -1228,6 +1269,9 @@ void TableSchema::MergeFrom(const TableSchema& from) {
   if (from.dimension() != 0) {
     set_dimension(from.dimension());
   }
+  if (from.index_file_size() != 0) {
+    set_index_file_size(from.index_file_size());
+  }
 }
 
 void TableSchema::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
@@ -1253,6 +1297,7 @@ void TableSchema::InternalSwap(TableSchema* other) {
   _internal_metadata_.Swap(&other->_internal_metadata_);
   swap(table_name_, other->table_name_);
   swap(dimension_, other->dimension_);
+  swap(index_file_size_, other->index_file_size_);
 }
 
 ::PROTOBUF_NAMESPACE_ID::Metadata TableSchema::GetMetadata() const {
@@ -5331,16 +5376,9 @@ const char* Index::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::inte
           CHK_(ptr);
         } else goto handle_unusual;
         continue;
-      // int32 index_file_size = 3;
+      // int32 metric_type = 3;
       case 3:
         if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) {
-          index_file_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
-          CHK_(ptr);
-        } else goto handle_unusual;
-        continue;
-      // int32 metric_type = 4;
-      case 4:
-        if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) {
           metric_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
           CHK_(ptr);
         } else goto handle_unusual;
@@ -5401,23 +5439,10 @@ bool Index::MergePartialFromCodedStream(
         break;
       }
 
-      // int32 index_file_size = 3;
+      // int32 metric_type = 3;
       case 3: {
         if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) {
 
-          DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
-                   ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>(
-                 input, &index_file_size_)));
-        } else {
-          goto handle_unusual;
-        }
-        break;
-      }
-
-      // int32 metric_type = 4;
-      case 4: {
-        if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) {
-
           DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
                    ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>(
                  input, &metric_type_)));
@@ -5464,14 +5489,9 @@ void Index::SerializeWithCachedSizes(
     ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(2, this->nlist(), output);
   }
 
-  // int32 index_file_size = 3;
-  if (this->index_file_size() != 0) {
-    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->index_file_size(), output);
-  }
-
-  // int32 metric_type = 4;
+  // int32 metric_type = 3;
   if (this->metric_type() != 0) {
-    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(4, this->metric_type(), output);
+    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->metric_type(), output);
   }
 
   if (_internal_metadata_.have_unknown_fields()) {
@@ -5497,14 +5517,9 @@ void Index::SerializeWithCachedSizes(
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->nlist(), target);
   }
 
-  // int32 index_file_size = 3;
-  if (this->index_file_size() != 0) {
-    target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->index_file_size(), target);
-  }
-
-  // int32 metric_type = 4;
+  // int32 metric_type = 3;
   if (this->metric_type() != 0) {
-    target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(4, this->metric_type(), target);
+    target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->metric_type(), target);
   }
 
   if (_internal_metadata_.have_unknown_fields()) {
@@ -5542,14 +5557,7 @@ size_t Index::ByteSizeLong() const {
         this->nlist());
   }
 
-  // int32 index_file_size = 3;
-  if (this->index_file_size() != 0) {
-    total_size += 1 +
-      ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
-        this->index_file_size());
-  }
-
-  // int32 metric_type = 4;
+  // int32 metric_type = 3;
   if (this->metric_type() != 0) {
     total_size += 1 +
       ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
@@ -5589,9 +5597,6 @@ void Index::MergeFrom(const Index& from) {
   if (from.nlist() != 0) {
     set_nlist(from.nlist());
   }
-  if (from.index_file_size() != 0) {
-    set_index_file_size(from.index_file_size());
-  }
   if (from.metric_type() != 0) {
     set_metric_type(from.metric_type());
   }
@@ -5620,7 +5625,6 @@ void Index::InternalSwap(Index* other) {
   _internal_metadata_.Swap(&other->_internal_metadata_);
   swap(index_type_, other->index_type_);
   swap(nlist_, other->nlist_);
-  swap(index_file_size_, other->index_file_size_);
   swap(metric_type_, other->metric_type_);
 }
 

cpp/src/grpc/gen-milvus/milvus.pb.h

@@ -396,6 +396,7 @@ class TableSchema :
   enum : int {
     kTableNameFieldNumber = 1,
     kDimensionFieldNumber = 2,
+    kIndexFileSizeFieldNumber = 3,
   };
   // .milvus.grpc.TableName table_name = 1;
   bool has_table_name() const;
@@ -410,6 +411,11 @@ class TableSchema :
   ::PROTOBUF_NAMESPACE_ID::int64 dimension() const;
   void set_dimension(::PROTOBUF_NAMESPACE_ID::int64 value);
 
+  // int64 index_file_size = 3;
+  void clear_index_file_size();
+  ::PROTOBUF_NAMESPACE_ID::int64 index_file_size() const;
+  void set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 value);
+
   // @@protoc_insertion_point(class_scope:milvus.grpc.TableSchema)
  private:
   class _Internal;
@@ -417,6 +423,7 @@ class TableSchema :
   ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
   ::milvus::grpc::TableName* table_name_;
   ::PROTOBUF_NAMESPACE_ID::int64 dimension_;
+  ::PROTOBUF_NAMESPACE_ID::int64 index_file_size_;
   mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
   friend struct ::TableStruct_milvus_2eproto;
 };
@@ -2318,8 +2325,7 @@ class Index :
   enum : int {
     kIndexTypeFieldNumber = 1,
     kNlistFieldNumber = 2,
-    kIndexFileSizeFieldNumber = 3,
-    kMetricTypeFieldNumber = 4,
+    kMetricTypeFieldNumber = 3,
   };
   // int32 index_type = 1;
   void clear_index_type();
@@ -2331,12 +2337,7 @@ class Index :
   ::PROTOBUF_NAMESPACE_ID::int32 nlist() const;
   void set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value);
 
-  // int32 index_file_size = 3;
-  void clear_index_file_size();
-  ::PROTOBUF_NAMESPACE_ID::int32 index_file_size() const;
-  void set_index_file_size(::PROTOBUF_NAMESPACE_ID::int32 value);
-
-  // int32 metric_type = 4;
+  // int32 metric_type = 3;
   void clear_metric_type();
   ::PROTOBUF_NAMESPACE_ID::int32 metric_type() const;
   void set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value);
@@ -2348,7 +2349,6 @@ class Index :
   ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
   ::PROTOBUF_NAMESPACE_ID::int32 index_type_;
   ::PROTOBUF_NAMESPACE_ID::int32 nlist_;
-  ::PROTOBUF_NAMESPACE_ID::int32 index_file_size_;
   ::PROTOBUF_NAMESPACE_ID::int32 metric_type_;
   mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
   friend struct ::TableStruct_milvus_2eproto;
@@ -2820,6 +2820,20 @@ inline void TableSchema::set_dimension(::PROTOBUF_NAMESPACE_ID::int64 value) {
   // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.dimension)
 }
 
+// int64 index_file_size = 3;
+inline void TableSchema::clear_index_file_size() {
+  index_file_size_ = PROTOBUF_LONGLONG(0);
+}
+inline ::PROTOBUF_NAMESPACE_ID::int64 TableSchema::index_file_size() const {
+  // @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.index_file_size)
+  return index_file_size_;
+}
+inline void TableSchema::set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 value) {
+  
+  index_file_size_ = value;
+  // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.index_file_size)
+}
+
 // -------------------------------------------------------------------
 
 // Range
@@ -3841,21 +3855,7 @@ inline void Index::set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value) {
   // @@protoc_insertion_point(field_set:milvus.grpc.Index.nlist)
 }
 
-// int32 index_file_size = 3;
-inline void Index::clear_index_file_size() {
-  index_file_size_ = 0;
-}
-inline ::PROTOBUF_NAMESPACE_ID::int32 Index::index_file_size() const {
-  // @@protoc_insertion_point(field_get:milvus.grpc.Index.index_file_size)
-  return index_file_size_;
-}
-inline void Index::set_index_file_size(::PROTOBUF_NAMESPACE_ID::int32 value) {
-  
-  index_file_size_ = value;
-  // @@protoc_insertion_point(field_set:milvus.grpc.Index.index_file_size)
-}
-
-// int32 metric_type = 4;
+// int32 metric_type = 3;
 inline void Index::clear_metric_type() {
   metric_type_ = 0;
 }

cpp/src/grpc/milvus.proto

@@ -19,6 +19,7 @@ message TableName {
 message TableSchema {
     TableName table_name = 1;
     int64 dimension = 2;
+    int64 index_file_size = 3;
 }
 
 /**
@@ -33,7 +34,7 @@ message Range {
  * @brief Record inserted
  */
 message RowRecord {
-    repeated float vector_data = 1;             //binary in thrift
+    repeated float vector_data = 1;             //binary vector data
 }
 
 /**
@@ -127,8 +128,7 @@ message Command {
 message Index {
     int32 index_type = 1;
     int32 nlist = 2;
-    int32 index_file_size = 3;
-    int32 metric_type = 4;
+    int32 metric_type = 3;
 }
 
 /**

cpp/src/scheduler/ResourceMgr.cpp

@@ -95,6 +95,13 @@ ResourceMgr::Stop() {
     }
 }
 
+void
+ResourceMgr::Clear() {
+    std::lock_guard<std::mutex> lck(resources_mutex_);
+    disk_resources_.clear();
+    resources_.clear();
+}
+
 void
 ResourceMgr::PostEvent(const EventPtr &event) {
     std::lock_guard<std::mutex> lock(event_mutex_);

cpp/src/scheduler/ResourceMgr.h

@@ -68,6 +68,9 @@ public:
     void
     Stop();
 
+    void
+    Clear();
+
     void
     PostEvent(const EventPtr &event);
 

cpp/src/scheduler/SchedInst.cpp

@@ -5,7 +5,8 @@
  ******************************************************************************/
 
 #include "SchedInst.h"
-
+#include "server/ServerConfig.h"
+#include "ResourceFactory.h"
 
 namespace zilliz {
 namespace milvus {
@@ -17,6 +18,40 @@ std::mutex ResMgrInst::mutex_;
 SchedulerPtr SchedInst::instance = nullptr;
 std::mutex SchedInst::mutex_;
 
+void
+SchedServInit() {
+    server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE);
+    auto resources = config.GetChild(server::CONFIG_RESOURCES).GetChildren();
+    for (auto &resource : resources) {
+        auto &resname = resource.first;
+        auto &resconf = resource.second;
+        auto type = resconf.GetValue(server::CONFIG_RESOURCE_TYPE);
+//        auto memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_MEMORY);
+        auto device_id = resconf.GetInt64Value(server::CONFIG_RESOURCE_DEVICE_ID);
+        auto enable_loader = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_LOADER);
+        auto enable_executor = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_EXECUTOR);
+
+        ResMgrInst::GetInstance()->Add(ResourceFactory::Create(resname,
+                                                               type,
+                                                               device_id,
+                                                               enable_loader,
+                                                               enable_executor));
+    }
+
+    auto default_connection = Connection("default_connection", 500.0);
+    auto connections = config.GetSequence(server::CONFIG_RESOURCE_CONNECTIONS);
+    for (auto &conn : connections) {
+        std::string delimiter = "===";
+        std::string left = conn.substr(0, conn.find(delimiter));
+        std::string right = conn.substr(conn.find(delimiter) + 3, conn.length());
+
+        ResMgrInst::GetInstance()->Connect(left, right, default_connection);
+    }
+
+    ResMgrInst::GetInstance()->Start();
+    SchedInst::GetInstance()->Start();
+}
+
 }
 }
 }

cpp/src/scheduler/SchedInst.h

@@ -52,6 +52,9 @@ private:
     static std::mutex mutex_;
 };
 
+void
+SchedServInit();
+
 }
 }
 }

cpp/src/scheduler/action/PushTaskToNeighbour.cpp

@@ -5,6 +5,7 @@
  ******************************************************************************/
 
 #include <list>
+#include <random>
 #include "Action.h"
 
 
@@ -38,6 +39,22 @@ push_task_round_robin(TaskTable &self_task_table, std::list<ResourcePtr> &neighb
     }
 }
 
+void
+push_task_randomly(TaskTable &self_task_table, std::vector<ResourcePtr> &neighbours) {
+    std::random_device rd;
+    std::mt19937 mt(rd());
+    std::uniform_int_distribution<uint64_t> dist(0, neighbours.size() - 1);
+    CacheMgr cache;
+
+    auto indexes = PickToMove(self_task_table, cache, self_task_table.Size());
+    for (auto index : indexes) {
+        if (self_task_table.Move(index)) {
+            auto task = self_task_table.Get(index)->task;
+            neighbours[dist(mt)]->task_table().Put(task);
+        }
+    }
+}
+
 void
 Action::PushTaskToNeighbour(const ResourceWPtr &res) {
     auto self = res.lock();
@@ -60,18 +77,21 @@ Action::PushTaskToNeighbourHasExecutor(const ResourceWPtr &res) {
     auto self = res.lock();
     if (not self) return;
 
-    std::list<ResourcePtr> neighbours;
+    std::list<ResourcePtr> l_neighbours;
+    std::vector<ResourcePtr> v_neighbours;
     for (auto &neighbour_node : self->GetNeighbours()) {
         auto node = neighbour_node.neighbour_node.lock();
         if (not node) continue;
 
         auto resource = std::static_pointer_cast<Resource>(node);
         if (resource->HasExecutor()) {
-            neighbours.emplace_back(resource);
+            l_neighbours.push_back(resource);
+            v_neighbours.push_back(resource);
         }
     }
 
-    push_task_round_robin(self->task_table(), neighbours);
+//    push_task_round_robin(self->task_table(), l_neighbours);
+    push_task_randomly(self->task_table(), v_neighbours);
 }
 
 

cpp/src/scheduler/task/SearchTask.cpp

@@ -84,7 +84,9 @@ CollectFileMetrics(int file_type, size_t file_size) {
 XSearchTask::XSearchTask(TableFileSchemaPtr file) : file_(file) {
     index_engine_ = EngineFactory::Build(file_->dimension_,
                                          file_->location_,
-                                         (EngineType) file_->engine_type_);
+                                         (EngineType) file_->engine_type_,
+                                         (MetricType)file_->metric_type_,
+                                         file_->nlist_);
 }
 
 void

cpp/src/sdk/CMakeLists.txt

@@ -12,43 +12,25 @@ include_directories(/usr/include)
 include_directories(include)
 include_directories(/usr/local/include)
 
-if (MILVUS_WITH_THRIFT STREQUAL "ON")
-    aux_source_directory(thrift thrift_client_files)
-    include_directories(thrift)
-    include_directories(${CMAKE_SOURCE_DIR}/src/thrift/gen-cpp)
 
-    set(thrift_service_files
-            ${CMAKE_SOURCE_DIR}/src/thrift/gen-cpp/MilvusService.cpp
-            ${CMAKE_SOURCE_DIR}/src/thrift/gen-cpp/milvus_constants.cpp
-            ${CMAKE_SOURCE_DIR}/src/thrift/gen-cpp/milvus_types.cpp
-            )
+aux_source_directory(grpc grpc_client_files)
 
-    add_library(milvus_sdk STATIC
-            ${interface_files}
-            ${thrift_client_files}
-            ${util_files}
-            ${thrift_service_files}
-            )
-else()
-    aux_source_directory(grpc grpc_client_files)
+include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus)
+include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-status)
 
-    include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus)
-    include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-status)
+set(grpc_service_files
+        ${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus/milvus.grpc.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus/milvus.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/grpc/gen-status/status.grpc.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/grpc/gen-status/status.pb.cc
+        )
 
-    set(grpc_service_files
-            ${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus/milvus.grpc.pb.cc
-            ${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus/milvus.pb.cc
-            ${CMAKE_SOURCE_DIR}/src/grpc/gen-status/status.grpc.pb.cc
-            ${CMAKE_SOURCE_DIR}/src/grpc/gen-status/status.pb.cc
-            )
-
-    add_library(milvus_sdk STATIC
-            ${interface_files}
-            ${grpc_client_files}
-            ${util_files}
-            ${grpc_service_files}
-            )
-endif()
+add_library(milvus_sdk STATIC
+        ${interface_files}
+        ${grpc_client_files}
+        ${util_files}
+        ${grpc_service_files}
+        )
 
 target_link_libraries(milvus_sdk
         ${third_party_libs}

cpp/src/sdk/examples/CMakeLists.txt

@@ -4,8 +4,4 @@
 # Proprietary and confidential.
 #-------------------------------------------------------------------------------
 
-if (MILVUS_WITH_THRIFT STREQUAL "ON")
-    add_subdirectory(thriftsimple)
-else()
-    add_subdirectory(grpcsimple)
-endif()
+add_subdirectory(grpcsimple)

cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp

@@ -22,6 +22,7 @@ namespace {
 
     const std::string TABLE_NAME = GetTableName();
     constexpr int64_t TABLE_DIMENSION = 512;
+    constexpr int64_t TABLE_INDEX_FILE_SIZE = 768;
     constexpr int64_t BATCH_ROW_COUNT = 100000;
     constexpr int64_t NQ = 10;
     constexpr int64_t TOP_K = 10;
@@ -94,6 +95,7 @@ namespace {
         TableSchema tb_schema;
         tb_schema.table_name = TABLE_NAME;
         tb_schema.dimension = TABLE_DIMENSION;
+        tb_schema.index_file_size = TABLE_INDEX_FILE_SIZE;
 
         return tb_schema;
     }
@@ -286,7 +288,6 @@ ClientTest::Test(const std::string& address, const std::string& port) {
         index.table_name = TABLE_NAME;
         index.index_type = IndexType::gpu_ivfflat;
         index.nlist = 1000;
-        index.index_file_size = 1024;
         index.metric_type = 1;
         Status stat = conn->CreateIndex(index);
         std::cout << "CreateIndex function call status: " << stat.ToString() << std::endl;

cpp/src/sdk/examples/thriftsimple/CMakeLists.txt

@@ -1,24 +0,0 @@
-#-------------------------------------------------------------------------------
-# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
-# Unauthorized copying of this file, via any medium is strictly prohibited.
-# Proprietary and confidential.
-#-------------------------------------------------------------------------------
-
-aux_source_directory(src src_files)
-
-include_directories(src)
-include_directories(../../include)
-
-link_directories(${CMAKE_BINARY_DIR})
-
-add_executable(sdk_simple
-        ./main.cpp
-        ${src_files}
-        )
-
-target_link_libraries(sdk_simple
-        milvus_sdk
-        pthread
-        )
-
-install(TARGETS sdk_simple DESTINATION bin)

cpp/src/sdk/examples/thriftsimple/main.cpp

@@ -1,68 +0,0 @@
-////////////////////////////////////////////////////////////////////////////////
-// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
-// Unauthorized copying of this file, via any medium is strictly prohibited.
-// Proprietary and confidential.
-////////////////////////////////////////////////////////////////////////////////
-
-#include <getopt.h>
-#include <libgen.h>
-#include <cstring>
-#include <string>
-
-#include "src/ClientTest.h"
-
-void print_help(const std::string &app_name);
-
-
-int
-main(int argc, char *argv[]) {
-    printf("Client start...\n");
-
-    std::string app_name = basename(argv[0]);
-    static struct option long_options[] = {{"server", optional_argument, 0, 's'},
-                                           {"port", optional_argument, 0, 'p'},
-                                           {"help", no_argument, 0, 'h'},
-                                           {NULL, 0, 0, 0}};
-
-    int option_index = 0;
-    std::string address = "127.0.0.1", port = "19530";
-    app_name = argv[0];
-
-    int value;
-    while ((value = getopt_long(argc, argv, "s:p:h", long_options, &option_index)) != -1) {
-        switch (value) {
-            case 's': {
-                char *address_ptr = strdup(optarg);
-                address = address_ptr;
-                free(address_ptr);
-                break;
-            }
-            case 'p': {
-                char *port_ptr = strdup(optarg);
-                port = port_ptr;
-                free(port_ptr);
-                break;
-            }
-            case 'h':
-            default:
-                print_help(app_name);
-                return EXIT_SUCCESS;
-        }
-    }
-
-    ClientTest test;
-    test.Test(address, port);
-
-    printf("Client stop...\n");
-    return 0;
-}
-
-void
-print_help(const std::string &app_name) {
-    printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
-    printf("  Options:\n");
-    printf("   -s --server   Server address, default 127.0.0.1\n");
-    printf("   -p --port     Server port, default 19530\n");
-    printf("   -h --help     Print help information\n");
-    printf("\n");
-}

cpp/src/sdk/examples/thriftsimple/src/ClientTest.cpp

@@ -1,288 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#include "ClientTest.h"
-#include "MilvusApi.h"
-
-#include <iostream>
-#include <time.h>
-#include <chrono>
-#include <unistd.h>
-
-using namespace ::milvus;
-
-namespace {
-    std::string GetTableName();
-
-    static const std::string TABLE_NAME = GetTableName();
-    constexpr int64_t TABLE_DIMENSION = 512;
-    constexpr int64_t BATCH_ROW_COUNT = 100000;
-    constexpr int64_t NQ = 10;
-    constexpr int64_t TOP_K = 10;
-    constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different
-    constexpr int64_t ADD_VECTOR_LOOP = 5;
-    constexpr int64_t SECONDS_EACH_HOUR = 3600;
-
-#define BLOCK_SPLITER std::cout << "===========================================" << std::endl;
-
-    void PrintTableSchema(const TableSchema& tb_schema) {
-        BLOCK_SPLITER
-        std::cout << "Table name: " << tb_schema.table_name << std::endl;
-        std::cout << "Table index type: " << (int)tb_schema.index_type << std::endl;
-        std::cout << "Table dimension: " << tb_schema.dimension << std::endl;
-        std::cout << "Table store raw data: " << (tb_schema.store_raw_vector ? "true" : "false") << std::endl;
-        BLOCK_SPLITER
-    }
-
-    void PrintSearchResult(const std::vector<std::pair<int64_t, RowRecord>>& search_record_array,
-            const std::vector<TopKQueryResult>& topk_query_result_array) {
-        BLOCK_SPLITER
-        std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl;
-
-        int32_t index = 0;
-        for(auto& result : topk_query_result_array) {
-            auto search_id = search_record_array[index].first;
-            index++;
-            std::cout << "No." << std::to_string(index) << " vector " << std::to_string(search_id)
-                << " top " << std::to_string(result.query_result_arrays.size())
-                << " search result:" << std::endl;
-            for(auto& item : result.query_result_arrays) {
-                std::cout << "\t" << std::to_string(item.id) << "\tdistance:" << std::to_string(item.distance);
-                std::cout << std::endl;
-            }
-        }
-
-        BLOCK_SPLITER
-    }
-
-    std::string CurrentTime() {
-        time_t tt;
-        time( &tt );
-        tt = tt + 8*SECONDS_EACH_HOUR;
-        tm* t= gmtime( &tt );
-
-        std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1)
-                          + "_" + std::to_string(t->tm_mday) + "_" + std::to_string(t->tm_hour)
-                          + "_" + std::to_string(t->tm_min) + "_" + std::to_string(t->tm_sec);
-
-        return str;
-    }
-
-    std::string CurrentTmDate(int64_t offset_day = 0) {
-        time_t tt;
-        time( &tt );
-        tt = tt + 8*SECONDS_EACH_HOUR;
-        tt = tt + 24*SECONDS_EACH_HOUR*offset_day;
-        tm* t= gmtime( &tt );
-
-        std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1)
-                          + "-" + std::to_string(t->tm_mday);
-
-        return str;
-    }
-
-    std::string GetTableName() {
-        static std::string s_id(CurrentTime());
-        return "tbl_" + s_id;
-    }
-
-    TableSchema BuildTableSchema() {
-        TableSchema tb_schema;
-        tb_schema.table_name = TABLE_NAME;
-        tb_schema.index_type = IndexType::gpu_ivfflat;
-        tb_schema.dimension = TABLE_DIMENSION;
-        tb_schema.store_raw_vector = true;
-
-        return tb_schema;
-    }
-
-    void BuildVectors(int64_t from, int64_t to,
-                      std::vector<RowRecord>& vector_record_array) {
-        if(to <= from){
-            return;
-        }
-
-        vector_record_array.clear();
-        for (int64_t k = from; k < to; k++) {
-            RowRecord record;
-            record.data.resize(TABLE_DIMENSION);
-            for(int64_t i = 0; i < TABLE_DIMENSION; i++) {
-                record.data[i] = (float)(k%(i+1));
-            }
-
-            vector_record_array.emplace_back(record);
-        }
-    }
-
-    void Sleep(int seconds) {
-        std::cout << "Waiting " << seconds << " seconds ..." << std::endl;
-        sleep(seconds);
-    }
-
-    class TimeRecorder {
-    public:
-        TimeRecorder(const std::string& title)
-        : title_(title) {
-            start_ = std::chrono::system_clock::now();
-        }
-
-        ~TimeRecorder() {
-            std::chrono::system_clock::time_point end = std::chrono::system_clock::now();
-            long span = (std::chrono::duration_cast<std::chrono::milliseconds> (end - start_)).count();
-            std::cout << title_ << " totally cost: " << span << " ms" << std::endl;
-        }
-
-    private:
-        std::string title_;
-        std::chrono::system_clock::time_point start_;
-    };
-
-    void CheckResult(const std::vector<std::pair<int64_t, RowRecord>>& search_record_array,
-                     const std::vector<TopKQueryResult>& topk_query_result_array) {
-        BLOCK_SPLITER
-        int64_t index = 0;
-        for(auto& result : topk_query_result_array) {
-            auto result_id = result.query_result_arrays[0].id;
-            auto search_id = search_record_array[index++].first;
-            if(result_id != search_id) {
-                std::cout << "The top 1 result is wrong: " << result_id
-                    << " vs. " << search_id << std::endl;
-            } else {
-                std::cout << "No." << index-1 << " Check result successfully" << std::endl;
-            }
-        }
-        BLOCK_SPLITER
-    }
-
-    void DoSearch(std::shared_ptr<Connection> conn,
-                  const std::vector<std::pair<int64_t, RowRecord>>& search_record_array,
-                  const std::string& phase_name) {
-        std::vector<Range> query_range_array;
-        Range rg;
-        rg.start_value = CurrentTmDate();
-        rg.end_value = CurrentTmDate(1);
-        query_range_array.emplace_back(rg);
-
-        std::vector<RowRecord> record_array;
-        for(auto& pair : search_record_array) {
-            record_array.push_back(pair.second);
-        }
-
-        std::vector<TopKQueryResult> topk_query_result_array;
-        {
-            TimeRecorder rc(phase_name);
-            Status stat = conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array);
-            std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
-        }
-
-        PrintSearchResult(search_record_array, topk_query_result_array);
-        CheckResult(search_record_array, topk_query_result_array);
-    }
-}
-
-void
-ClientTest::Test(const std::string& address, const std::string& port) {
-    std::shared_ptr<Connection> conn = Connection::Create();
-
-    {//connect server
-        ConnectParam param = {address, port};
-        Status stat = conn->Connect(param);
-        std::cout << "Connect function call status: " << stat.ToString() << std::endl;
-    }
-
-    {//server version
-        std::string version = conn->ServerVersion();
-        std::cout << "Server version: " << version << std::endl;
-    }
-
-    {//sdk version
-        std::string version = conn->ClientVersion();
-        std::cout << "SDK version: " << version << std::endl;
-    }
-
-    {
-        std::vector<std::string> tables;
-        Status stat = conn->ShowTables(tables);
-        std::cout << "ShowTables function call status: " << stat.ToString() << std::endl;
-        std::cout << "All tables: " << std::endl;
-        for(auto& table : tables) {
-            int64_t row_count = 0;
-            stat = conn->CountTable(table, row_count);
-            std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl;
-        }
-    }
-
-    {//create table
-        TableSchema tb_schema = BuildTableSchema();
-        Status stat = conn->CreateTable(tb_schema);
-        std::cout << "CreateTable function call status: " << stat.ToString() << std::endl;
-        PrintTableSchema(tb_schema);
-
-        bool has_table = conn->HasTable(tb_schema.table_name);
-        if(has_table) {
-            std::cout << "Table is created" << std::endl;
-        }
-    }
-
-    {//describe table
-        TableSchema tb_schema;
-        Status stat = conn->DescribeTable(TABLE_NAME, tb_schema);
-        std::cout << "DescribeTable function call status: " << stat.ToString() << std::endl;
-        PrintTableSchema(tb_schema);
-    }
-
-    //add vectors
-    std::vector<std::pair<int64_t, RowRecord>> search_record_array;
-    for (int i = 0; i < ADD_VECTOR_LOOP; i++) {
-        TimeRecorder recorder("Add vector No." + std::to_string(i));
-        std::vector<RowRecord> record_array;
-        int64_t begin_index = i * BATCH_ROW_COUNT;
-        BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array);
-        std::vector<int64_t> record_ids;
-        Status stat = conn->Insert(TABLE_NAME, record_array, record_ids);
-        std::cout << "AddVector function call status: " << stat.ToString() << std::endl;
-        std::cout << "Returned id array count: " << record_ids.size() << std::endl;
-
-        if(i == 0) {
-            for(int64_t k = SEARCH_TARGET; k < SEARCH_TARGET + NQ; k++) {
-                search_record_array.push_back(
-                        std::make_pair(record_ids[k], record_array[k]));
-            }
-        }
-    }
-
-    {//search vectors without index
-        Sleep(2);
-        DoSearch(conn, search_record_array, "Search without index");
-    }
-
-    {//wait unit build index finish
-        TimeRecorder recorder("Build index");
-        std::cout << "Wait until build all index done" << std::endl;
-        IndexParam index_param;
-        index_param.table_name = TABLE_NAME;
-        Status stat = conn->CreateIndex(index_param);
-        std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl;
-    }
-
-    {//search vectors after build index finish
-        DoSearch(conn, search_record_array, "Search after build index finish");
-    }
-
-    {//delete table
-        Status stat = conn->DropTable(TABLE_NAME);
-        std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl;
-    }
-
-    {//server status
-        std::string status = conn->ServerStatus();
-        std::cout << "Server status before disconnect: " << status << std::endl;
-    }
-    Connection::Destroy(conn);
-    {//server status
-        std::string status = conn->ServerStatus();
-        std::cout << "Server status after disconnect: " << status << std::endl;
-    }
-}

cpp/src/sdk/examples/thriftsimple/src/ClientTest.h

@@ -1,13 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#pragma once
-
-#include <string>
-
-class ClientTest {
-public:
-    void Test(const std::string& address, const std::string& port);
-};

cpp/src/sdk/grpc/ClientProxy.cpp

@@ -83,6 +83,7 @@ ClientProxy::CreateTable(const TableSchema &param) {
         ::milvus::grpc::TableSchema schema;
         schema.mutable_table_name()->set_table_name(param.table_name);
         schema.set_dimension(param.dimension);
+        schema.set_index_file_size(param.index_file_size);
 
         return client_ptr_->CreateTable(schema);
     } catch (std::exception &ex) {
@@ -119,7 +120,6 @@ ClientProxy::CreateIndex(const IndexParam &index_param) {
                 index_param.table_name);
         grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type);
         grpc_index_param.mutable_index()->set_nlist(index_param.nlist);
-        grpc_index_param.mutable_index()->set_index_file_size(index_param.index_file_size);
         grpc_index_param.mutable_index()->set_metric_type(index_param.metric_type);
         return client_ptr_->CreateIndex(grpc_index_param);
 
@@ -273,6 +273,7 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
 
         table_schema.table_name = grpc_schema.table_name().table_name();
         table_schema.dimension = grpc_schema.dimension();
+        table_schema.index_file_size = grpc_schema.index_file_size();
 
         return status;
     } catch (std::exception &ex) {
@@ -363,7 +364,6 @@ ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_para
         Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param);
         index_param.index_type = (IndexType)(grpc_index_param.mutable_index()->index_type());
         index_param.nlist = grpc_index_param.mutable_index()->nlist();
-        index_param.index_file_size = grpc_index_param.mutable_index()->index_file_size();
         index_param.metric_type = grpc_index_param.mutable_index()->metric_type();
 
         return status;

cpp/src/sdk/include/MilvusApi.h

@@ -35,9 +35,8 @@ struct ConnectParam {
  */
 struct TableSchema {
     std::string table_name;                                ///< Table name
-    IndexType index_type = IndexType::invalid;             ///< Index type
     int64_t dimension = 0;                                 ///< Vector dimension, must be a positive value
-    bool store_raw_vector = false;                          ///< Is vector raw data stored in the table
+    int64_t index_file_size = 0;                           ///< Index file size, must be a positive value
 };
 
 /**
@@ -78,7 +77,6 @@ struct IndexParam {
     std::string table_name;
     IndexType index_type;
     int32_t nlist;
-    int32_t index_file_size;
     int32_t metric_type;
 };
 

cpp/src/sdk/interface/ConnectionImpl.h

@@ -6,11 +6,7 @@
 #pragma once
 
 #include "MilvusApi.h"
-#ifdef MILVUS_ENABLE_THRIFT
-#include "src/sdk/thrift/ClientProxy.h"
-#else
 #include "src/sdk/grpc/ClientProxy.h"
-#endif
 
 namespace milvus {
 

cpp/src/sdk/thrift/ClientProxy.cpp

@@ -1,346 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#include "ClientProxy.h"
-
-namespace milvus {
-
-std::shared_ptr<ThriftClient>&
-ClientProxy::ClientPtr() const {
-    if(client_ptr == nullptr) {
-        client_ptr = std::make_shared<ThriftClient>();
-    }
-    return client_ptr;
-}
-
-bool ClientProxy::IsConnected() const {
-    return (client_ptr != nullptr && connected_);
-}
-
-Status
-ClientProxy::Connect(const ConnectParam &param) {
-    Disconnect();
-
-    int32_t port = atoi(param.port.c_str());
-    Status status = ClientPtr()->Connect(param.ip_address, port, THRIFT_PROTOCOL_BINARY);
-    if(status.ok()) {
-        connected_ = true;
-    }
-
-    return status;
-}
-
-Status
-ClientProxy::Connect(const std::string &uri) {
-    Disconnect();
-
-    size_t index = uri.find_first_of(":", 0);
-    if((index == std::string::npos)) {
-        return Status::Invalid("Invalid uri");
-    }
-
-    ConnectParam param;
-    param.ip_address = uri.substr(0, index);
-    param.port = uri.substr(index + 1);
-
-    return Connect(param);
-}
-
-Status
-ClientProxy::Connected() const {
-    if(!IsConnected()) {
-        return Status(StatusCode::NotConnected, "not connected to server");
-    }
-
-    try {
-        std::string info;
-        ClientPtr()->interface()->Ping(info, "");
-    }  catch ( std::exception& ex) {
-        return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what()));
-    }
-
-    return Status::OK();
-}
-
-Status
-ClientProxy::Disconnect() {
-    if(!IsConnected()) {
-        return Status(StatusCode::NotConnected, "not connected to server");
-    }
-
-    connected_ = false;
-    return ClientPtr()->Disconnect();
-}
-
-std::string
-ClientProxy::ClientVersion() const {
-    return "";
-}
-
-Status
-ClientProxy::CreateTable(const TableSchema &param) {
-    if(!IsConnected()) {
-        return Status(StatusCode::NotConnected, "not connected to server");
-    }
-
-    try {
-
-        thrift::TableSchema schema;
-        schema.__set_table_name(param.table_name);
-        schema.__set_index_type((int)param.index_type);
-        schema.__set_dimension(param.dimension);
-        schema.__set_store_raw_vector(param.store_raw_vector);
-
-        ClientPtr()->interface()->CreateTable(schema);
-
-    }  catch ( std::exception& ex) {
-        return Status(StatusCode::UnknownError, "failed to create table: " + std::string(ex.what()));
-    }
-
-    return Status::OK();
-}
-
-bool
-ClientProxy::HasTable(const std::string &table_name) {
-    if(!IsConnected()) {
-        return false;
-    }
-
-    return ClientPtr()->interface()->HasTable(table_name);
-}
-
-Status
-ClientProxy::DropTable(const std::string &table_name) {
-    if(!IsConnected()) {
-        return Status(StatusCode::NotConnected, "not connected to server");
-    }
-
-    try {
-        ClientPtr()->interface()->DeleteTable(table_name);
-
-    }  catch ( std::exception& ex) {
-        return Status(StatusCode::UnknownError, "failed to delete table: " + std::string(ex.what()));
-    }
-
-    return Status::OK();
-}
-
-Status
-ClientProxy::CreateIndex(const IndexParam &index_param) {
-    if(!IsConnected()) {
-        return Status(StatusCode::NotConnected, "not connected to server");
-    }
-
-    try {
-        ClientPtr()->interface()->BuildIndex(index_param.table_name);
-
-    }  catch ( std::exception& ex) {
-        return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what()));
-    }
-
-    return Status::OK();
-}
-
-Status
-ClientProxy::Insert(const std::string &table_name,
-                          const std::vector<RowRecord> &record_array,
-                          std::vector<int64_t> &id_array) {
-    if(!IsConnected()) {
-        return Status(StatusCode::NotConnected, "not connected to server");
-    }
-
-    try {
-        std::vector<thrift::RowRecord> thrift_records;
-        for(auto& record : record_array) {
-            thrift::RowRecord thrift_record;
-
-            thrift_record.vector_data.resize(record.data.size() * sizeof(double));
-            double *dbl = (double *) (const_cast<char *>(thrift_record.vector_data.data()));
-            for (size_t i = 0; i < record.data.size(); i++) {
-                dbl[i] = (double) (record.data[i]);
-            }
-
-            thrift_records.emplace_back(thrift_record);
-        }
-        ClientPtr()->interface()->AddVector(id_array, table_name, thrift_records);
-
-    }  catch ( std::exception& ex) {
-        return Status(StatusCode::UnknownError, "failed to add vector: " + std::string(ex.what()));
-    }
-
-    return Status::OK();
-}
-
-Status
-ClientProxy::Search(const std::string &table_name,
-                          const std::vector<RowRecord> &query_record_array,
-                          const std::vector<Range> &query_range_array,
-                          int64_t topk,
-                          std::vector<TopKQueryResult> &topk_query_result_array) {
-    if(!IsConnected()) {
-        return Status(StatusCode::NotConnected, "not connected to server");
-    }
-
-    try {
-
-        //step 1: convert vectors data
-        std::vector<thrift::RowRecord> thrift_records;
-        for(auto& record : query_record_array) {
-            thrift::RowRecord thrift_record;
-
-            thrift_record.vector_data.resize(record.data.size() * sizeof(double));
-            auto dbl = (double *) (const_cast<char *>(thrift_record.vector_data.data()));
-            for (size_t i = 0; i < record.data.size(); i++) {
-                dbl[i] = (double) (record.data[i]);
-            }
-
-            thrift_records.emplace_back(thrift_record);
-        }
-
-        //step 2: convert range array
-        std::vector<thrift::Range> thrift_ranges;
-        for(auto& range : query_range_array) {
-            thrift::Range thrift_range;
-            thrift_range.__set_start_value(range.start_value);
-            thrift_range.__set_end_value(range.end_value);
-
-            thrift_ranges.emplace_back(thrift_range);
-        }
-
-        //step 3: search vectors
-        std::vector<thrift::TopKQueryBinResult> result_array;
-        ClientPtr()->interface()->SearchVector2(result_array, table_name, thrift_records, thrift_ranges, topk);
-
-        //step 4: convert result array
-        for(auto& thrift_topk_result : result_array) {
-            TopKQueryResult result;
-
-            size_t id_count = thrift_topk_result.id_array.size()/sizeof(int64_t);
-            size_t dist_count = thrift_topk_result.distance_array.size()/ sizeof(double);
-            if(id_count != dist_count) {
-                return Status(StatusCode::UnknownError, "illegal result");
-            }
-
-            auto id_ptr = (int64_t*)thrift_topk_result.id_array.data();
-            auto dist_ptr = (double*)thrift_topk_result.distance_array.data();
-            for(size_t i = 0; i < id_count; i++) {
-                QueryResult query_result;
-                query_result.id = id_ptr[i];
-                query_result.distance = dist_ptr[i];
-                result.query_result_arrays.emplace_back(query_result);
-            }
-
-            topk_query_result_array.emplace_back(result);
-        }
-
-    }  catch ( std::exception& ex) {
-        return Status(StatusCode::UnknownError, "failed to search vectors: " + std::string(ex.what()));
-    }
-
-    return Status::OK();
-}
-
-Status
-ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_schema) {
-    if(!IsConnected()) {
-        return Status(StatusCode::NotConnected, "not connected to server");
-    }
-
-    try {
-        thrift::TableSchema thrift_schema;
-        ClientPtr()->interface()->DescribeTable(thrift_schema, table_name);
-
-        table_schema.table_name = thrift_schema.table_name;
-        table_schema.index_type = (IndexType)thrift_schema.index_type;
-        table_schema.dimension = thrift_schema.dimension;
-        table_schema.store_raw_vector = thrift_schema.store_raw_vector;
-
-    }  catch ( std::exception& ex) {
-        return Status(StatusCode::UnknownError, "failed to describe table: " + std::string(ex.what()));
-    }
-
-    return Status::OK();
-}
-
-Status
-ClientProxy::CountTable(const std::string &table_name, int64_t &row_count) {
-    if(!IsConnected()) {
-        return Status(StatusCode::NotConnected, "not connected to server");
-    }
-
-    try {
-        row_count = ClientPtr()->interface()->GetTableRowCount(table_name);
-
-    }  catch ( std::exception& ex) {
-        return Status(StatusCode::UnknownError, "failed to show tables: " + std::string(ex.what()));
-    }
-
-    return Status::OK();
-}
-
-Status
-ClientProxy::ShowTables(std::vector<std::string> &table_array) {
-    if(!IsConnected()) {
-        return Status(StatusCode::NotConnected, "not connected to server");
-    }
-
-    try {
-        ClientPtr()->interface()->ShowTables(table_array);
-
-    }  catch ( std::exception& ex) {
-        return Status(StatusCode::UnknownError, "failed to show tables: " + std::string(ex.what()));
-    }
-
-    return Status::OK();
-}
-
-std::string
-ClientProxy::ServerVersion() const {
-    if(!IsConnected()) {
-        return "";
-    }
-
-    try {
-        std::string version;
-        ClientPtr()->interface()->Ping(version, "version");
-        return version;
-    }  catch ( std::exception& ex) {
-        return "";
-    }
-}
-
-std::string
-ClientProxy::ServerStatus() const {
-    if(!IsConnected()) {
-        return "not connected to server";
-    }
-
-    try {
-        std::string dummy;
-        ClientPtr()->interface()->Ping(dummy, "");
-        return "server alive";
-    }  catch ( std::exception& ex) {
-        return "connection lost";
-    }
-}
-
-Status ClientProxy::DeleteByRange(Range &range, const std::string &table_name) {
-    return Status::OK();
-}
-
-Status ClientProxy::PreloadTable(const std::string &table_name) const {
-    return Status::OK();
-}
-
-Status ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_param) const {
-    index_param.table_name = table_name;
-    return index_param;
-}
-
-Status ClientProxy::DropIndex(const std::string &table_name) const {
-    return Status::OK();
-}
-    
-}

cpp/src/sdk/thrift/ClientProxy.h

@@ -1,72 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#pragma once
-
-#include "MilvusApi.h"
-#include "ThriftClient.h"
-
-namespace milvus {
-
-class ClientProxy : public Connection {
-public:
-    // Implementations of the Connection interface
-    virtual Status Connect(const ConnectParam &param) override;
-
-    virtual Status Connect(const std::string &uri) override;
-
-    virtual Status Connected() const override;
-
-    virtual Status Disconnect() override;
-
-    virtual Status CreateTable(const TableSchema &param) override;
-
-    virtual bool HasTable(const std::string &table_name) override;
-
-    virtual Status DropTable(const std::string &table_name) override;
-
-    virtual Status CreateIndex(const IndexParam &index_param) override;
-
-    virtual Status Insert(const std::string &table_name,
-                             const std::vector<RowRecord> &record_array,
-                             std::vector<int64_t> &id_array) override;
-
-    virtual Status Search(const std::string &table_name,
-                                const std::vector<RowRecord> &query_record_array,
-                                const std::vector<Range> &query_range_array,
-                                int64_t topk,
-                                std::vector<TopKQueryResult> &topk_query_result_array) override;
-
-    virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override;
-
-    virtual Status CountTable(const std::string &table_name, int64_t &row_count) override;
-
-    virtual Status ShowTables(std::vector<std::string> &table_array) override;
-
-    virtual std::string ClientVersion() const override;
-
-    virtual std::string ServerVersion() const override;
-
-    virtual std::string ServerStatus() const override;
-
-    virtual Status DeleteByRange(Range &range, const std::string &table_name) override;
-
-    virtual Status PreloadTable(const std::string &table_name) const override;
-
-    virtual Status DescribeIndex(const std::string &table_name, IndexParam &index_param) const override;
-
-    virtual Status DropIndex(const std::string &table_name) const override;
-
-private:
-    std::shared_ptr<ThriftClient>& ClientPtr() const;
-
-    bool IsConnected() const;
-
-private:
-    mutable std::shared_ptr<ThriftClient> client_ptr;
-    bool connected_ = false;
-};
-
-}

cpp/src/sdk/thrift/ThriftClient.cpp

@@ -1,102 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#include "ThriftClient.h"
-
-#include "milvus_types.h"
-#include "milvus_constants.h"
-
-#include <exception>
-
-#include <thrift/protocol/TBinaryProtocol.h>
-#include <thrift/protocol/TJSONProtocol.h>
-#include <thrift/protocol/TCompactProtocol.h>
-#include <thrift/protocol/TDebugProtocol.h>
-#include <thrift/server/TSimpleServer.h>
-#include <thrift/server/TThreadPoolServer.h>
-#include <thrift/transport/TServerSocket.h>
-#include <thrift/transport/TSocket.h>
-#include <thrift/transport/TBufferTransports.h>
-#include <thrift/concurrency/PosixThreadFactory.h>
-
-namespace milvus {
-
-using namespace ::apache::thrift;
-using namespace ::apache::thrift::protocol;
-using namespace ::apache::thrift::transport;
-using namespace ::apache::thrift::concurrency;
-
-ThriftClient::ThriftClient() {
-
-}
-
-ThriftClient::~ThriftClient() {
-
-}
-
-ServiceClientPtr
-ThriftClient::interface() {
-    if(client_ == nullptr) {
-        throw std::exception();
-    }
-    return client_;
-}
-
-Status
-ThriftClient::Connect(const std::string& address, int32_t port, const std::string& protocol) {
-    try {
-        stdcxx::shared_ptr<TSocket> socket_ptr(new transport::TSocket(address, port));
-        stdcxx::shared_ptr<TTransport> transport_ptr(new TBufferedTransport(socket_ptr));
-        stdcxx::shared_ptr<TProtocol> protocol_ptr;
-        if(protocol == THRIFT_PROTOCOL_BINARY) {
-            protocol_ptr.reset(new TBinaryProtocol(transport_ptr));
-        } else if(protocol == THRIFT_PROTOCOL_JSON) {
-            protocol_ptr.reset(new TJSONProtocol(transport_ptr));
-        } else if(protocol == THRIFT_PROTOCOL_COMPACT) {
-            protocol_ptr.reset(new TCompactProtocol(transport_ptr));
-        } else {
-            //CLIENT_LOG_ERROR << "Service protocol: " << protocol << " is not supported currently";
-            return Status(StatusCode::InvalidAgument, "unsupported protocol");
-        }
-
-        transport_ptr->open();
-        client_ = std::make_shared<thrift::MilvusServiceClient>(protocol_ptr);
-    } catch ( std::exception& ex) {
-        //CLIENT_LOG_ERROR << "connect encounter exception: " << ex.what();
-        return Status(StatusCode::NotConnected, "failed to connect server" + std::string(ex.what()));
-    }
-
-    return Status::OK();
-}
-Status
-ThriftClient::Disconnect() {
-    try {
-        if(client_ != nullptr) {
-            auto protocol = client_->getInputProtocol();
-            if(protocol != nullptr) {
-                auto transport = protocol->getTransport();
-                if(transport != nullptr) {
-                    transport->close();
-                }
-            }
-        }
-    } catch ( std::exception& ex) {
-        //CLIENT_LOG_ERROR << "disconnect encounter exception: " << ex.what();
-        return Status(StatusCode::UnknownError, "failed to disconnect: " + std::string(ex.what()));
-    }
-
-    return Status::OK();
-}
-
-/////////////////////////////////////////////////////////////////////////////////////////////////////////
-ThriftClientSession::ThriftClientSession(const std::string& address, int32_t port, const std::string& protocol) {
-    Connect(address, port, protocol);
-}
-
-ThriftClientSession::~ThriftClientSession() {
-    Disconnect();
-}
-
-}

cpp/src/sdk/thrift/ThriftClient.h

@@ -1,42 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#pragma once
-
-#include "MilvusService.h"
-#include "Status.h"
-
-#include <memory>
-
-namespace milvus {
-
-using ServiceClientPtr = std::shared_ptr<::milvus::thrift::MilvusServiceClient>;
-
-static const char* THRIFT_PROTOCOL_JSON = "json";
-static const char* THRIFT_PROTOCOL_BINARY = "binary";
-static const char* THRIFT_PROTOCOL_COMPACT = "compact";
-
-class ThriftClient {
-public:
-    ThriftClient();
-    virtual ~ThriftClient();
-
-    ServiceClientPtr interface();
-
-    Status Connect(const std::string& address, int32_t port, const std::string& protocol);
-    Status Disconnect();
-
-private:
-    ServiceClientPtr client_;
-};
-
-
-class ThriftClientSession : public ThriftClient {
-public:
-    ThriftClientSession(const std::string& address, int32_t port, const std::string& protocol);
-    ~ThriftClientSession();
-};
-
-}

cpp/src/server/DBWrapper.cpp

@@ -27,19 +27,6 @@ DBWrapper::DBWrapper() {
     std::string db_slave_path = db_config.GetValue(CONFIG_DB_SLAVE_PATH);
     StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths);
 
-    int64_t index_size = db_config.GetInt64Value(CONFIG_DB_INDEX_TRIGGER_SIZE);
-    if(index_size > 0) {//ensure larger than zero, unit is MB
-        opt.index_trigger_size = (size_t)index_size * engine::ONE_MB;
-    }
-    int64_t insert_buffer_size = db_config.GetInt64Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4);
-    if (insert_buffer_size >= 1) {
-        opt.insert_buffer_size = insert_buffer_size * engine::ONE_GB;
-    }
-    else {
-        std::cout << "ERROR: insert_buffer_size should be at least 1 GB" << std::endl;
-        kill(0, SIGUSR1);
-    }
-
     // cache config
     ConfigNode& cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE);
     opt.insert_cache_immediately_ = cache_config.GetBoolValue(CONFIG_INSERT_CACHE_IMMEDIATELY, false);

cpp/src/server/Server.cpp

@@ -5,13 +5,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 #include <thread>
 #include "Server.h"
-//#include "ServerConfig.h"
-#ifdef MILVUS_ENABLE_THRIFT
-#include "server/thrift_impl/MilvusServer.h"
-#else
 #include "server/grpc_impl/GrpcMilvusServer.h"
-#endif
-
 #include "utils/Log.h"
 #include "utils/SignalUtil.h"
 #include "utils/TimeRecorder.h"
@@ -24,6 +18,7 @@
 //#include <numaif.h>
 #include <unistd.h>
 #include <string.h>
+#include <src/scheduler/SchedInst.h>
 
 #include "metrics/Metrics.h"
 
@@ -163,6 +158,7 @@ Server::Start() {
             signal(SIGTERM, SignalUtil::HandleSignal);
             server::Metrics::GetInstance().Init();
             server::SystemInfo::GetInstance().Init();
+            engine::SchedServInit();
             std::cout << "Milvus server start successfully." << std::endl;
             StartService();
 
@@ -225,20 +221,12 @@ Server::LoadConfig() {
 
 void
 Server::StartService() {
-#ifdef MILVUS_ENABLE_THRIFT
-    MilvusServer::StartService();
-#else
     grpc::GrpcMilvusServer::StartService();
-#endif
 }
 
 void
 Server::StopService() {
-#ifdef MILVUS_ENABLE_THRIFT
-    MilvusServer::StopService();
-#else
     grpc::GrpcMilvusServer::StopService();
-#endif
 }
 
 }

cpp/src/server/ServerConfig.cpp

@@ -79,19 +79,6 @@ ServerError ServerConfig::ValidateConfig() const {
         return SERVER_INVALID_ARGUMENT;
     }
 
-    uint64_t index_building_threshold = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INDEX_TRIGGER_SIZE, 1024);
-    index_building_threshold *= MB;
-
-    size_t gpu_mem = 0;
-    ValidationUtil::GetGpuMemory(gpu_index, gpu_mem);
-    if(index_building_threshold >= gpu_mem) {
-        std::cout << "Error: index_building_threshold execeed gpu memory" << std::endl;
-        return SERVER_INVALID_ARGUMENT;
-    } else if(index_building_threshold >= gpu_mem/3) {
-        std::cout << "Warnning: index_building_threshold is greater than 1/3 of gpu memory, "
-            << "some index type(such as IVFLAT) may cause cuda::bad_alloc() error" << std::endl;
-    }
-
     //cache config validation
     ConfigNode cache_config = GetConfig(CONFIG_CACHE);
     uint64_t cache_cap = (uint64_t)cache_config.GetInt64Value(CONFIG_CPU_CACHE_CAPACITY, 16);

cpp/src/server/ServerConfig.h

@@ -24,7 +24,6 @@ static const char* CONFIG_DB = "db_config";
 static const char* CONFIG_DB_URL = "db_backend_url";
 static const char* CONFIG_DB_PATH = "db_path";
 static const char* CONFIG_DB_SLAVE_PATH = "db_slave_path";
-static const char* CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold";
 static const char* CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold";
 static const char* CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold";
 static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
@@ -50,6 +49,16 @@ static const std::string CONFIG_ENGINE = "engine_config";
 static const std::string CONFIG_DCBT = "use_blas_threshold";
 static const std::string CONFIG_OMP_THREAD_NUM = "omp_thread_num";
 
+static const char* CONFIG_RESOURCE = "resource_config";
+static const char* CONFIG_RESOURCES = "resources";
+static const char* CONFIG_RESOURCE_TYPE = "type";
+static const char* CONFIG_RESOURCE_MEMORY = "memory";
+static const char* CONFIG_RESOURCE_DEVICE_ID = "device_id";
+static const char* CONFIG_RESOURCE_ENABLE_LOADER = "enable_loader";
+static const char* CONFIG_RESOURCE_ENABLE_EXECUTOR = "enable_executor";
+static const char* CONFIG_RESOURCE_CONNECTIONS = "connections";
+
+
 class ServerConfig {
  public:
     static ServerConfig &GetInstance();

cpp/src/server/grpc_impl/GrpcRequestTask.cpp

@@ -133,10 +133,16 @@ CreateTableTask::OnExecute() {
             return SetError(res, "Invalid table dimension: " + std::to_string(schema_.dimension()));
         }
 
+        res = ValidationUtil::ValidateTableIndexFileSize(schema_.index_file_size());
+        if(res != SERVER_SUCCESS) {
+            return SetError(res, "Invalid index file size: " + std::to_string(schema_.index_file_size()));
+        }
+
         //step 2: construct table schema
         engine::meta::TableSchema table_info;
-        table_info.dimension_ = (uint16_t) schema_.dimension();
         table_info.table_id_ = schema_.table_name().table_name();
+        table_info.dimension_ = (uint16_t) schema_.dimension();
+        table_info.index_file_size_ = schema_.index_file_size();
 
         //step 3: create table
         engine::Status stat = DBWrapper::DB()->CreateTable(table_info);
@@ -245,16 +251,10 @@ CreateIndexTask::OnExecute() {
             return SetError(res, "Invalid index metric type: " + std::to_string(index_param_.mutable_index()->metric_type()));
         }
 
-        res = ValidationUtil::ValidateTableIndexFileSize(index_param_.mutable_index()->index_file_size());
-        if(res != SERVER_SUCCESS) {
-            return SetError(res, "Invalid index file size: " + std::to_string(index_param_.mutable_index()->index_file_size()));
-        }
-
         //step 2: check table existence
         engine::TableIndex index;
         index.engine_type_ = index_param_.mutable_index()->index_type();
         index.nlist_ = index_param_.mutable_index()->nlist();
-        index.index_file_size_ = index_param_.mutable_index()->index_file_size();
         index.metric_type_ = index_param_.mutable_index()->metric_type();
         stat = DBWrapper::DB()->CreateIndex(table_name_, index);
         if (!stat.ok()) {
@@ -616,23 +616,20 @@ SearchTask::OnExecute() {
         auto record_array_size = search_param_.query_record_array_size();
         std::vector<float> vec_f(record_array_size * table_info.dimension_, 0);
         for (size_t i = 0; i < record_array_size; i++) {
-            for (size_t j = 0; j < table_info.dimension_; j++) {
-                if (search_param_.query_record_array(i).vector_data().empty()) {
-                    return SetError(SERVER_INVALID_ROWRECORD_ARRAY,
-                                    "Query record float array is empty");
-                }
-                uint64_t query_vec_dim = search_param_.query_record_array(
-                        i).vector_data().size();
-                if (query_vec_dim != table_info.dimension_) {
-                    ServerError error_code = SERVER_INVALID_VECTOR_DIMENSION;
-                    std::string error_msg =
-                            "Invalid rowrecord dimension: " + std::to_string(query_vec_dim)
-                            + " vs. table dimension:" + std::to_string(table_info.dimension_);
-                    return SetError(error_code, error_msg);
-                }
-                vec_f[i * table_info.dimension_ + j] = search_param_.query_record_array(
-                        i).vector_data(j);
+            if (search_param_.query_record_array(i).vector_data().empty()) {
+                return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Query record float array is empty");
             }
+            uint64_t query_vec_dim = search_param_.query_record_array(i).vector_data().size();
+            if (query_vec_dim != table_info.dimension_) {
+                ServerError error_code = SERVER_INVALID_VECTOR_DIMENSION;
+                std::string error_msg = "Invalid rowrecord dimension: " + std::to_string(query_vec_dim)
+                                        + " vs. table dimension:" + std::to_string(table_info.dimension_);
+                return SetError(error_code, error_msg);
+            }
+
+            memcpy(&vec_f[i * table_info.dimension_],
+                   search_param_.query_record_array(i).vector_data().data(),
+                   table_info.dimension_ * sizeof(float));
         }
         rc.ElapseFromBegin("prepare vector data");
 
@@ -903,7 +900,6 @@ DescribeIndexTask::OnExecute() {
         index_param_.mutable_table_name()->set_table_name(table_name_);
         index_param_.mutable_index()->set_index_type(index.engine_type_);
         index_param_.mutable_index()->set_nlist(index.nlist_);
-        index_param_.mutable_index()->set_index_file_size(index.index_file_size_);
         index_param_.mutable_index()->set_metric_type(index.metric_type_);
 
         rc.ElapseFromBegin("totally cost");

cpp/src/server/thrift_impl/MilvusServer.cpp

@@ -1,106 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#include "MilvusServer.h"
-#include "RequestHandler.h"
-#include "src/server/ServerConfig.h"
-#include "ThreadPoolServer.h"
-#include "src/server/DBWrapper.h"
-#include "utils/Log.h"
-
-#include "milvus_types.h"
-#include "milvus_constants.h"
-#include "faiss/utils.h"
-
-#include <thrift/protocol/TBinaryProtocol.h>
-#include <thrift/protocol/TJSONProtocol.h>
-#include <thrift/protocol/TDebugProtocol.h>
-#include <thrift/protocol/TCompactProtocol.h>
-#include <thrift/server/TSimpleServer.h>
-#include <thrift/server/TThreadPoolServer.h>
-#include <thrift/transport/TServerSocket.h>
-#include <thrift/transport/TBufferTransports.h>
-#include <thrift/concurrency/PosixThreadFactory.h>
-
-#include <thread>
-#include <iostream>
-
-//extern int distance_compute_blas_threshold;
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-using namespace ::milvus::thrift;
-using namespace ::apache::thrift;
-using namespace ::apache::thrift::protocol;
-using namespace ::apache::thrift::transport;
-using namespace ::apache::thrift::server;
-using namespace ::apache::thrift::concurrency;
-
-static stdcxx::shared_ptr<TServer> s_server;
-
-void
-MilvusServer::StartService() {
-    if(s_server != nullptr){
-        StopService();
-    }
-
-    ServerConfig &config = ServerConfig::GetInstance();
-    ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
-    ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE);
-    std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, "127.0.0.1");
-    int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, 19530);
-    std::string protocol = server_config.GetValue(CONFIG_SERVER_PROTOCOL, "binary");
-    faiss::distance_compute_blas_threshold = engine_config.GetInt32Value(CONFIG_DCBT,20);
-//    std::cout<<"distance_compute_blas_threshold = "<< faiss::distance_compute_blas_threshold << std::endl;
-    try {
-        DBWrapper::DB();//initialize db
-
-        stdcxx::shared_ptr<RequestHandler> handler(new RequestHandler());
-        stdcxx::shared_ptr<TProcessor> processor(new MilvusServiceProcessor(handler));
-        stdcxx::shared_ptr<TServerTransport> server_transport(new TServerSocket(address, port));
-        stdcxx::shared_ptr<TTransportFactory> transport_factory(new TBufferedTransportFactory());
-
-        stdcxx::shared_ptr<TProtocolFactory> protocol_factory;
-        if (protocol == "binary") {
-            protocol_factory.reset(new TBinaryProtocolFactory());
-        } else if (protocol == "json") {
-            protocol_factory.reset(new TJSONProtocolFactory());
-        } else if (protocol == "compact") {
-            protocol_factory.reset(new TCompactProtocolFactory());
-        } else {
-            // SERVER_LOG_INFO << "Service protocol: " << protocol << " is not supported currently";
-            return;
-        }
-
-        stdcxx::shared_ptr<ThreadManager> threadManager(ThreadManager::newSimpleThreadManager(16));
-        stdcxx::shared_ptr<PosixThreadFactory> threadFactory(new PosixThreadFactory());
-        threadManager->threadFactory(threadFactory);
-        threadManager->start();
-
-        s_server.reset(new ThreadPoolServer(processor,
-                                             server_transport,
-                                             transport_factory,
-                                             protocol_factory,
-                                             threadManager));
-        s_server->serve();
-
-    } catch (apache::thrift::TException& ex) {
-        std::cout << "ERROR! " << ex.what() << std::endl;
-        kill(0, SIGUSR1);
-    }
-}
-
-void
-MilvusServer::StopService() {
-    if(s_server != nullptr) {
-        s_server->stop();
-    }
-}
-
-}
-}
-}

cpp/src/server/thrift_impl/MilvusServer.h

@@ -1,23 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#pragma once
-
-#include <cstdint>
-#include <string>
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-class MilvusServer {
-public:
-    static void StartService();
-    static void StopService();
-};
-
-}
-}
-}

cpp/src/server/thrift_impl/RequestHandler.cpp

@@ -1,124 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-
-#include "RequestHandler.h"
-#include "RequestTask.h"
-#include "utils/TimeRecorder.h"
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-using namespace ::milvus;
-
-RequestHandler::RequestHandler() {
-
-}
-
-void
-RequestHandler::CreateTable(const thrift::TableSchema &param) {
-    BaseTaskPtr task_ptr = CreateTableTask::Create(param);
-    RequestScheduler::ExecTask(task_ptr);
-}
-
-bool
-RequestHandler::HasTable(const std::string &table_name) {
-    bool has_table = false;
-    BaseTaskPtr task_ptr = HasTableTask::Create(table_name, has_table);
-    RequestScheduler::ExecTask(task_ptr);
-
-    return has_table;
-}
-
-void
-RequestHandler::DeleteTable(const std::string &table_name) {
-    BaseTaskPtr task_ptr = DeleteTableTask::Create(table_name);
-    RequestScheduler::ExecTask(task_ptr);
-}
-
-void
-RequestHandler::BuildIndex(const std::string &table_name) {
-    BaseTaskPtr task_ptr = BuildIndexTask::Create(table_name);
-    RequestScheduler::ExecTask(task_ptr);
-}
-
-void
-RequestHandler::AddVector(std::vector<int64_t> &_return,
-                              const std::string &table_name,
-                              const std::vector<thrift::RowRecord> &record_array) {
-    BaseTaskPtr task_ptr = AddVectorTask::Create(table_name, record_array, _return);
-    RequestScheduler::ExecTask(task_ptr);
-}
-
-void
-RequestHandler::SearchVector(std::vector<thrift::TopKQueryResult> &_return,
-                                 const std::string &table_name,
-                                 const std::vector<thrift::RowRecord> &query_record_array,
-                                 const std::vector<thrift::Range> &query_range_array,
-                                 const int64_t topk) {
-//    SERVER_LOG_DEBUG << "Entering RequestHandler::SearchVector";
-    BaseTaskPtr task_ptr = SearchVectorTask1::Create(table_name, std::vector<std::string>(), query_record_array,
-            query_range_array, topk, _return);
-    RequestScheduler::ExecTask(task_ptr);
-}
-
-void
-RequestHandler::SearchVector2(std::vector<thrift::TopKQueryBinResult> & _return,
-                                  const std::string& table_name,
-                                  const std::vector<thrift::RowRecord> & query_record_array,
-                                  const std::vector<thrift::Range> & query_range_array,
-                                  const int64_t topk) {
-    BaseTaskPtr task_ptr = SearchVectorTask2::Create(table_name, std::vector<std::string>(), query_record_array,
-                                                    query_range_array, topk, _return);
-    RequestScheduler::ExecTask(task_ptr);
-}
-
-void
-RequestHandler::SearchVectorInFiles(std::vector<::milvus::thrift::TopKQueryResult> &_return,
-                                        const std::string& table_name,
-                                        const std::vector<std::string> &file_id_array,
-                                        const std::vector<::milvus::thrift::RowRecord> &query_record_array,
-                                        const std::vector<::milvus::thrift::Range> &query_range_array,
-                                        const int64_t topk) {
-//    SERVER_LOG_DEBUG << "Entering RequestHandler::SearchVectorInFiles. file_id_array size = " << std::to_string(file_id_array.size());
-    BaseTaskPtr task_ptr = SearchVectorTask1::Create(table_name, file_id_array, query_record_array,
-            query_range_array, topk, _return);
-    RequestScheduler::ExecTask(task_ptr);
-}
-
-void
-RequestHandler::DescribeTable(thrift::TableSchema &_return, const std::string &table_name) {
-    BaseTaskPtr task_ptr = DescribeTableTask::Create(table_name, _return);
-    RequestScheduler::ExecTask(task_ptr);
-}
-
-int64_t
-RequestHandler::GetTableRowCount(const std::string& table_name) {
-    int64_t row_count = 0;
-    {
-        BaseTaskPtr task_ptr = GetTableRowCountTask::Create(table_name, row_count);
-        RequestScheduler::ExecTask(task_ptr);
-        task_ptr->WaitToFinish();
-    }
-
-    return row_count;
-}
-
-void
-RequestHandler::ShowTables(std::vector<std::string> &_return) {
-    BaseTaskPtr task_ptr = ShowTablesTask::Create(_return);
-    RequestScheduler::ExecTask(task_ptr);
-}
-
-void
-RequestHandler::Ping(std::string& _return, const std::string& cmd) {
-    BaseTaskPtr task_ptr = PingTask::Create(cmd, _return);
-    RequestScheduler::ExecTask(task_ptr);
-}
-
-}
-}
-}

cpp/src/server/thrift_impl/RequestHandler.h

@@ -1,206 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#pragma once
-
-#include <cstdint>
-#include <string>
-
-#include "MilvusService.h"
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-class RequestHandler : virtual public ::milvus::thrift::MilvusServiceIf {
-public:
-    RequestHandler();
-
-    /**
-     * @brief Create table method
-     *
-     * This method is used to create table
-     *
-     * @param param, use to provide table information to be created.
-     *
-     *
-     * @param param
-     */
-    void CreateTable(const ::milvus::thrift::TableSchema &param);
-
-    /**
-     * @brief Test table existence method
-     *
-     * This method is used to test table existence.
-     *
-     * @param table_name, table name is going to be tested.
-     *
-     *
-     * @param table_name
-     */
-    bool HasTable(const std::string &table_name);
-
-    /**
-     * @brief Delete table method
-     *
-     * This method is used to delete table.
-     *
-     * @param table_name, table name is going to be deleted.
-     *
-     *
-     * @param table_name
-     */
-    void DeleteTable(const std::string& table_name);
-
-    /**
-     * @brief build index by table method
-     *
-     * This method is used to build index by table in sync.
-     *
-     * @param table_name, table name is going to be built index.
-     *
-     *
-     * @param table_name
-     */
-    void BuildIndex(const std::string &table_name);
-
-    /**
-     * @brief Add vector array to table
-     *
-     * This method is used to add vector array to table.
-     *
-     * @param table_name, table_name is inserted.
-     * @param record_array, vector array is inserted.
-     *
-     * @return vector id array
-     *
-     * @param table_name
-     * @param record_array
-     */
-    void AddVector(std::vector<int64_t> & _return,
-            const std::string& table_name,
-            const std::vector<::milvus::thrift::RowRecord> & record_array);
-
-    /**
-     * @brief Query vector
-     *
-     * This method is used to query vector in table.
-     *
-     * @param table_name, table_name is queried.
-     * @param query_record_array, all vector are going to be queried.
-     * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-     * @param topk, how many similarity vectors will be searched.
-     *
-     * @return query result array.
-     *
-     * @param table_name
-     * @param query_record_array
-     * @param query_range_array
-     * @param topk
-     */
-    void SearchVector(std::vector<::milvus::thrift::TopKQueryResult> & _return,
-            const std::string& table_name,
-            const std::vector<::milvus::thrift::RowRecord> & query_record_array,
-            const std::vector<::milvus::thrift::Range> & query_range_array,
-            const int64_t topk);
-
-    /**
-     * @brief Query vector
-     *
-     * This method is used to query vector in table.
-     *
-     * @param table_name, table_name is queried.
-     * @param query_record_array, all vector are going to be queried.
-     * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-     * @param topk, how many similarity vectors will be searched.
-     *
-     * @return query binary result array.
-     *
-     * @param table_name
-     * @param query_record_array
-     * @param query_range_array
-     * @param topk
-     */
-    void SearchVector2(std::vector<::milvus::thrift::TopKQueryBinResult> & _return,
-            const std::string& table_name,
-            const std::vector<::milvus::thrift::RowRecord> & query_record_array,
-            const std::vector<::milvus::thrift::Range> & query_range_array,
-            const int64_t topk);
-
-    /**
-   * @brief Internal use query interface
-   *
-   * This method is used to query vector in specified files.
-   *
-   * @param file_id_array, specified files id array, queried.
-   * @param query_record_array, all vector are going to be queried.
-   * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-   * @param topk, how many similarity vectors will be searched.
-   *
-   * @return query result array.
-   *
-   * @param file_id_array
-   * @param query_record_array
-   * @param query_range_array
-   * @param topk
-   */
-    virtual void SearchVectorInFiles(std::vector<::milvus::thrift::TopKQueryResult> & _return,
-            const std::string& table_name,
-            const std::vector<std::string> & file_id_array,
-            const std::vector<::milvus::thrift::RowRecord> & query_record_array,
-            const std::vector<::milvus::thrift::Range> & query_range_array,
-            const int64_t topk);
-
-    /**
-     * @brief Get table schema
-     *
-     * This method is used to get table schema.
-     *
-     * @param table_name, target table name.
-     *
-     * @return table schema
-     *
-     * @param table_name
-     */
-    void DescribeTable(::milvus::thrift::TableSchema& _return, const std::string& table_name);
-
-    /**
-     * @brief Get table row count
-     *
-     * This method is used to get table row count.
-     *
-     * @param table_name, target table name.
-     *
-     * @return table row count
-     *
-     * @param table_name
-     */
-    int64_t GetTableRowCount(const std::string& table_name);
-
-    /**
-     * @brief List all tables in database
-     *
-     * This method is used to list all tables.
-     *
-     *
-     * @return table names.
-     */
-    void ShowTables(std::vector<std::string> & _return);
-
-    /**
-     * @brief Give the server status
-     *
-     * This method is used to give the server status.
-     *
-     * @return Server status.
-     *
-     * @param cmd
-     */
-    void Ping(std::string& _return, const std::string& cmd);
-};
-
-}
-}
-}

cpp/src/server/thrift_impl/RequestScheduler.cpp

@@ -1,218 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#include "RequestScheduler.h"
-#include "utils/Log.h"
-
-#include "milvus_types.h"
-#include "milvus_constants.h"
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-using namespace ::milvus;
-
-namespace {
-    const std::map<ServerError, thrift::ErrorCode::type> &ErrorMap() {
-        static const std::map<ServerError, thrift::ErrorCode::type> code_map = {
-                {SERVER_UNEXPECTED_ERROR,         thrift::ErrorCode::UNEXPECTED_ERROR},
-                {SERVER_UNSUPPORTED_ERROR,        thrift::ErrorCode::UNEXPECTED_ERROR},
-                {SERVER_NULL_POINTER,             thrift::ErrorCode::UNEXPECTED_ERROR},
-                {SERVER_INVALID_ARGUMENT,         thrift::ErrorCode::ILLEGAL_ARGUMENT},
-                {SERVER_FILE_NOT_FOUND,           thrift::ErrorCode::FILE_NOT_FOUND},
-                {SERVER_NOT_IMPLEMENT,            thrift::ErrorCode::UNEXPECTED_ERROR},
-                {SERVER_BLOCKING_QUEUE_EMPTY,     thrift::ErrorCode::UNEXPECTED_ERROR},
-                {SERVER_CANNOT_CREATE_FOLDER,     thrift::ErrorCode::CANNOT_CREATE_FOLDER},
-                {SERVER_CANNOT_CREATE_FILE,       thrift::ErrorCode::CANNOT_CREATE_FILE},
-                {SERVER_CANNOT_DELETE_FOLDER,     thrift::ErrorCode::CANNOT_DELETE_FOLDER},
-                {SERVER_CANNOT_DELETE_FILE,       thrift::ErrorCode::CANNOT_DELETE_FILE},
-                {SERVER_TABLE_NOT_EXIST,          thrift::ErrorCode::TABLE_NOT_EXISTS},
-                {SERVER_INVALID_TABLE_NAME,       thrift::ErrorCode::ILLEGAL_TABLE_NAME},
-                {SERVER_INVALID_TABLE_DIMENSION,  thrift::ErrorCode::ILLEGAL_DIMENSION},
-                {SERVER_INVALID_TIME_RANGE,       thrift::ErrorCode::ILLEGAL_RANGE},
-                {SERVER_INVALID_VECTOR_DIMENSION, thrift::ErrorCode::ILLEGAL_DIMENSION},
-
-                {SERVER_INVALID_INDEX_TYPE,       thrift::ErrorCode::ILLEGAL_INDEX_TYPE},
-                {SERVER_INVALID_ROWRECORD,        thrift::ErrorCode::ILLEGAL_ROWRECORD},
-                {SERVER_INVALID_ROWRECORD_ARRAY,  thrift::ErrorCode::ILLEGAL_ROWRECORD},
-                {SERVER_INVALID_TOPK,             thrift::ErrorCode::ILLEGAL_TOPK},
-                {SERVER_ILLEGAL_VECTOR_ID,        thrift::ErrorCode::ILLEGAL_VECTOR_ID},
-                {SERVER_ILLEGAL_SEARCH_RESULT,    thrift::ErrorCode::ILLEGAL_SEARCH_RESULT},
-                {SERVER_CACHE_ERROR,              thrift::ErrorCode::CACHE_FAILED},
-                {DB_META_TRANSACTION_FAILED,      thrift::ErrorCode::META_FAILED},
-                {SERVER_BUILD_INDEX_ERROR,        thrift::ErrorCode::BUILD_INDEX_ERROR},
-        };
-
-        return code_map;
-    }
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-BaseTask::BaseTask(const std::string& task_group, bool async)
-    : task_group_(task_group),
-      async_(async),
-      done_(false),
-      error_code_(SERVER_SUCCESS) {
-
-}
-
-BaseTask::~BaseTask() {
-    WaitToFinish();
-}
-
-ServerError BaseTask::Execute() {
-    error_code_ = OnExecute();
-    done_ = true;
-    finish_cond_.notify_all();
-    return error_code_;
-}
-
-ServerError BaseTask::SetError(ServerError error_code, const std::string& error_msg) {
-    error_code_ = error_code;
-    error_msg_ = error_msg;
-
-    SERVER_LOG_ERROR << error_msg_;
-    return error_code_;
-}
-
-ServerError BaseTask::WaitToFinish() {
-    std::unique_lock <std::mutex> lock(finish_mtx_);
-    finish_cond_.wait(lock, [this] { return done_; });
-
-    return error_code_;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-RequestScheduler::RequestScheduler()
-: stopped_(false) {
-    Start();
-}
-
-RequestScheduler::~RequestScheduler() {
-    Stop();
-}
-
-void RequestScheduler::ExecTask(BaseTaskPtr& task_ptr) {
-    if(task_ptr == nullptr) {
-        return;
-    }
-
-    RequestScheduler& scheduler = RequestScheduler::GetInstance();
-    scheduler.ExecuteTask(task_ptr);
-
-    if(!task_ptr->IsAsync()) {
-        task_ptr->WaitToFinish();
-        ServerError err = task_ptr->ErrorCode();
-        if (err != SERVER_SUCCESS) {
-            thrift::Exception ex;
-            ex.__set_code(ErrorMap().at(err));
-            std::string msg = task_ptr->ErrorMsg();
-            if(msg.empty()){
-                msg = "Error message not set";
-            }
-            ex.__set_reason(msg);
-            throw ex;
-        }
-    }
-}
-
-void RequestScheduler::Start() {
-    if(!stopped_) {
-        return;
-    }
-
-    stopped_ = false;
-}
-
-void RequestScheduler::Stop() {
-    if(stopped_) {
-        return;
-    }
-
-    SERVER_LOG_INFO << "Scheduler gonna stop...";
-    {
-        std::lock_guard<std::mutex> lock(queue_mtx_);
-        for(auto iter : task_groups_) {
-            if(iter.second != nullptr) {
-                iter.second->Put(nullptr);
-            }
-        }
-    }
-
-    for(auto iter : execute_threads_) {
-        if(iter == nullptr)
-            continue;
-
-        iter->join();
-    }
-    stopped_ = true;
-    SERVER_LOG_INFO << "Scheduler stopped";
-}
-
-ServerError RequestScheduler::ExecuteTask(const BaseTaskPtr& task_ptr) {
-    if(task_ptr == nullptr) {
-        return SERVER_NULL_POINTER;
-    }
-
-    ServerError err = PutTaskToQueue(task_ptr);
-    if(err != SERVER_SUCCESS) {
-        return err;
-    }
-
-    if(task_ptr->IsAsync()) {
-        return SERVER_SUCCESS;//async execution, caller need to call WaitToFinish at somewhere
-    }
-
-    return task_ptr->WaitToFinish();//sync execution
-}
-
-namespace {
-    void TakeTaskToExecute(TaskQueuePtr task_queue) {
-        if(task_queue == nullptr) {
-            return;
-        }
-
-        while(true) {
-            BaseTaskPtr task = task_queue->Take();
-            if (task == nullptr) {
-                break;//stop the thread
-            }
-
-            try {
-                ServerError err = task->Execute();
-                if(err != SERVER_SUCCESS) {
-                    SERVER_LOG_ERROR << "Task failed with code: " << err;
-                }
-            } catch (std::exception& ex) {
-                SERVER_LOG_ERROR << "Task failed to execute: " << ex.what();
-            }
-        }
-    }
-}
-
-ServerError RequestScheduler::PutTaskToQueue(const BaseTaskPtr& task_ptr) {
-    std::lock_guard<std::mutex> lock(queue_mtx_);
-
-    std::string group_name = task_ptr->TaskGroup();
-    if(task_groups_.count(group_name) > 0) {
-        task_groups_[group_name]->Put(task_ptr);
-    } else {
-        TaskQueuePtr queue = std::make_shared<TaskQueue>();
-        queue->Put(task_ptr);
-        task_groups_.insert(std::make_pair(group_name, queue));
-
-        //start a thread
-        ThreadPtr thread = std::make_shared<std::thread>(&TakeTaskToExecute, queue);
-        execute_threads_.push_back(thread);
-        SERVER_LOG_INFO << "Create new thread for task group: " << group_name;
-    }
-
-    return SERVER_SUCCESS;
-}
-
-}
-}
-}

cpp/src/server/thrift_impl/RequestScheduler.h

@@ -1,88 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#pragma once
-
-#include "utils/BlockingQueue.h"
-
-#include <map>
-#include <vector>
-#include <thread>
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-class BaseTask {
-protected:
-    BaseTask(const std::string& task_group, bool async = false);
-    virtual ~BaseTask();
-
-public:
-    ServerError Execute();
-    ServerError WaitToFinish();
-
-    std::string TaskGroup() const { return task_group_; }
-
-    ServerError ErrorCode() const { return error_code_; }
-    std::string ErrorMsg() const { return error_msg_; }
-
-    bool IsAsync() const { return async_; }
-
-protected:
-    virtual ServerError OnExecute() = 0;
-
-    ServerError SetError(ServerError error_code, const std::string& msg);
-
-protected:
-    mutable std::mutex finish_mtx_;
-    std::condition_variable finish_cond_;
-
-    std::string task_group_;
-    bool async_;
-    bool done_;
-    ServerError error_code_;
-    std::string error_msg_;
-};
-
-using BaseTaskPtr = std::shared_ptr<BaseTask>;
-using TaskQueue = BlockingQueue<BaseTaskPtr>;
-using TaskQueuePtr = std::shared_ptr<TaskQueue>;
-using ThreadPtr = std::shared_ptr<std::thread>;
-
-class RequestScheduler {
-public:
-    static RequestScheduler& GetInstance() {
-        static RequestScheduler scheduler;
-        return scheduler;
-    }
-
-    void Start();
-    void Stop();
-
-    ServerError ExecuteTask(const BaseTaskPtr& task_ptr);
-
-    static void ExecTask(BaseTaskPtr& task_ptr);
-
-protected:
-    RequestScheduler();
-    virtual ~RequestScheduler();
-
-    ServerError PutTaskToQueue(const BaseTaskPtr& task_ptr);
-
-private:
-    mutable std::mutex queue_mtx_;
-
-    std::map<std::string, TaskQueuePtr> task_groups_;
-
-    std::vector<ThreadPtr> execute_threads_;
-
-    bool stopped_;
-};
-
-
-}
-}
-}

cpp/src/server/thrift_impl/RequestTask.cpp

@@ -1,779 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#include "RequestTask.h"
-#include "src/server/ServerConfig.h"
-#include "utils/CommonUtil.h"
-#include "utils/Log.h"
-#include "utils/TimeRecorder.h"
-#include "utils/ValidationUtil.h"
-#include "src/server/DBWrapper.h"
-#include "version.h"
-
-#ifdef MILVUS_ENABLE_PROFILING
-#include "gperftools/profiler.h"
-#endif
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-using namespace ::milvus;
-
-static const char* DQL_TASK_GROUP = "dql";
-static const char* DDL_DML_TASK_GROUP = "ddl_dml";
-static const char* PING_TASK_GROUP = "ping";
-
-using DB_META = zilliz::milvus::engine::meta::Meta;
-using DB_DATE = zilliz::milvus::engine::meta::DateT;
-
-namespace {
-    engine::EngineType EngineType(int type) {
-        static std::map<int, engine::EngineType> map_type = {
-                {0, engine::EngineType::INVALID},
-                {1, engine::EngineType::FAISS_IDMAP},
-                {2, engine::EngineType::FAISS_IVFFLAT},
-                {3, engine::EngineType::FAISS_IVFSQ8},
-                {4, engine::EngineType::NSG_MIX},
-        };
-
-        if(map_type.find(type) == map_type.end()) {
-            return engine::EngineType::INVALID;
-        }
-
-        return map_type[type];
-    }
-
-    int IndexType(engine::EngineType type) {
-        static std::map<engine::EngineType, int> map_type = {
-                {engine::EngineType::INVALID, 0},
-                {engine::EngineType::FAISS_IDMAP, 1},
-                {engine::EngineType::FAISS_IVFFLAT, 2},
-                {engine::EngineType::FAISS_IVFSQ8, 3},
-                {engine::EngineType::NSG_MIX, 4},
-        };
-
-        if(map_type.find(type) == map_type.end()) {
-            return 0;
-        }
-
-        return map_type[type];
-    }
-
-    void
-    ConvertRowRecordToFloatArray(const std::vector<thrift::RowRecord>& record_array,
-                                 uint64_t dimension,
-                                 std::vector<float>& float_array,
-                                 ServerError& error_code,
-                                 std::string& error_msg) {
-        uint64_t vec_count = record_array.size();
-        float_array.resize(vec_count*dimension);//allocate enough memory
-        for(uint64_t i = 0; i < vec_count; i++) {
-            const auto& record = record_array[i];
-            if(record.vector_data.empty()) {
-                error_code = SERVER_INVALID_ROWRECORD;
-                error_msg = "Rowrecord float array is empty";
-                return;
-            }
-            uint64_t vec_dim = record.vector_data.size()/sizeof(double);//how many double value?
-            if(vec_dim != dimension) {
-                error_code = SERVER_INVALID_VECTOR_DIMENSION;
-                error_msg = "Invalid rowrecord dimension: " + std::to_string(vec_dim)
-                                 + " vs. table dimension:" + std::to_string(dimension);
-                return;
-            }
-
-            //convert double array to float array(thrift has no float type)
-            const double* d_p = reinterpret_cast<const double*>(record.vector_data.data());
-            for(uint64_t d = 0; d < vec_dim; d++) {
-                float_array[i*vec_dim + d] = (float)(d_p[d]);
-            }
-        }
-    }
-
-    static constexpr long DAY_SECONDS = 86400;
-
-    void
-    ConvertTimeRangeToDBDates(const std::vector<thrift::Range> &range_array,
-                              std::vector<DB_DATE>& dates,
-                              ServerError& error_code,
-                              std::string& error_msg) {
-        dates.clear();
-        for(auto& range : range_array) {
-            time_t tt_start, tt_end;
-            tm tm_start, tm_end;
-            if(!CommonUtil::TimeStrToTime(range.start_value, tt_start, tm_start)){
-                error_code = SERVER_INVALID_TIME_RANGE;
-                error_msg = "Invalid time range: " + range.start_value;
-                return;
-            }
-
-            if(!CommonUtil::TimeStrToTime(range.end_value, tt_end, tm_end)){
-                error_code = SERVER_INVALID_TIME_RANGE;
-                error_msg = "Invalid time range: " + range.start_value;
-                return;
-            }
-
-            long days = (tt_end > tt_start) ? (tt_end - tt_start)/DAY_SECONDS : (tt_start - tt_end)/DAY_SECONDS;
-            if(days == 0) {
-                error_code = SERVER_INVALID_TIME_RANGE;
-                error_msg = "Invalid time range: " + range.start_value + " to " + range.end_value;
-                return ;
-            }
-
-            for(long i = 0; i < days; i++) {
-                time_t tt_day = tt_start + DAY_SECONDS*i;
-                tm tm_day;
-                CommonUtil::ConvertTime(tt_day, tm_day);
-
-                long date = tm_day.tm_year*10000 + tm_day.tm_mon*100 + tm_day.tm_mday;//according to db logic
-                dates.push_back(date);
-            }
-        }
-    }
-
-    std::string
-    GetCurrTimeStr() {
-        char tm_buf[20] = {0};
-        time_t tt;
-        time(&tt);
-        tt = tt + 8 * 60 * 60;
-        tm* t = gmtime(&tt);
-        sprintf(tm_buf, "%4d%02d%02d_%02d%02d%02d", (t->tm_year+1900), (t->tm_mon+1), (t->tm_mday),
-                                                    (t->tm_hour), (t->tm_min), (t->tm_sec));
-        return tm_buf;
-    }
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-CreateTableTask::CreateTableTask(const thrift::TableSchema& schema)
-: BaseTask(DDL_DML_TASK_GROUP),
-  schema_(schema) {
-
-}
-
-BaseTaskPtr CreateTableTask::Create(const thrift::TableSchema& schema) {
-    return std::shared_ptr<BaseTask>(new CreateTableTask(schema));
-}
-
-ServerError CreateTableTask::OnExecute() {
-    TimeRecorder rc("CreateTableTask");
-
-    try {
-        //step 1: check arguments
-        ServerError res = SERVER_SUCCESS;
-        res = ValidationUtil::ValidateTableName(schema_.table_name);
-        if(res != SERVER_SUCCESS) {
-            return SetError(res, "Invalid table name: " + schema_.table_name);
-        }
-
-        res = ValidationUtil::ValidateTableDimension(schema_.dimension);
-        if(res != SERVER_SUCCESS) {
-            return SetError(res, "Invalid table dimension: " + std::to_string(schema_.dimension));
-        }
-
-        res = ValidationUtil::ValidateTableIndexType(schema_.index_type);
-        if(res != SERVER_SUCCESS) {
-            return SetError(res, "Invalid index type: " + std::to_string(schema_.index_type));
-        }
-
-        //step 2: construct table schema
-        engine::meta::TableSchema table_info;
-        table_info.dimension_ = (uint16_t)schema_.dimension;
-        table_info.table_id_ = schema_.table_name;
-        table_info.engine_type_ = (int)EngineType(schema_.index_type);
-        table_info.store_raw_data_ = schema_.store_raw_vector;
-
-        //step 3: create table
-        engine::Status stat = DBWrapper::DB()->CreateTable(table_info);
-        if(!stat.ok()) {//table could exist
-            return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
-        }
-
-    } catch (std::exception& ex) {
-        SERVER_LOG_ERROR << "CreateTableTask encounter exception: " << ex.what();
-        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
-    }
-
-    rc.ElapseFromBegin("totally cost");
-
-    return SERVER_SUCCESS;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-DescribeTableTask::DescribeTableTask(const std::string &table_name, thrift::TableSchema &schema)
-    : BaseTask(DDL_DML_TASK_GROUP),
-      table_name_(table_name),
-      schema_(schema) {
-    schema_.table_name = table_name_;
-}
-
-BaseTaskPtr DescribeTableTask::Create(const std::string& table_name, thrift::TableSchema& schema) {
-    return std::shared_ptr<BaseTask>(new DescribeTableTask(table_name, schema));
-}
-
-ServerError DescribeTableTask::OnExecute() {
-    TimeRecorder rc("DescribeTableTask");
-
-    try {
-        //step 1: check arguments
-        ServerError res = SERVER_SUCCESS;
-        res = ValidationUtil::ValidateTableName(table_name_);
-        if(res != SERVER_SUCCESS) {
-            return SetError(res, "Invalid table name: " + table_name_);
-        }
-
-        //step 2: get table info
-        engine::meta::TableSchema table_info;
-        table_info.table_id_ = table_name_;
-        engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
-        if(!stat.ok()) {
-            return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
-        }
-
-        schema_.table_name = table_info.table_id_;
-        schema_.index_type = IndexType((engine::EngineType)table_info.engine_type_);
-        schema_.dimension = table_info.dimension_;
-        schema_.store_raw_vector = table_info.store_raw_data_;
-
-    } catch (std::exception& ex) {
-        SERVER_LOG_ERROR << "DescribeTableTask encounter exception: " << ex.what();
-        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
-    }
-
-    rc.ElapseFromBegin("totally cost");
-
-    return SERVER_SUCCESS;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-BuildIndexTask::BuildIndexTask(const std::string& table_name)
-    : BaseTask(DDL_DML_TASK_GROUP),
-      table_name_(table_name) {
-}
-
-BaseTaskPtr BuildIndexTask::Create(const std::string& table_name) {
-    return std::shared_ptr<BaseTask>(new BuildIndexTask(table_name));
-}
-
-ServerError BuildIndexTask::OnExecute() {
-    try {
-        TimeRecorder rc("BuildIndexTask");
-
-        //step 1: check arguments
-        ServerError res = SERVER_SUCCESS;
-        res = ValidationUtil::ValidateTableName(table_name_);
-        if(res != SERVER_SUCCESS) {
-            return SetError(res, "Invalid table name: " + table_name_);
-        }
-
-        bool has_table = false;
-        engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table);
-        if(!has_table) {
-            return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
-        }
-
-        //step 2: check table existence
-        stat = DBWrapper::DB()->BuildIndex(table_name_);
-        if(!stat.ok()) {
-            return SetError(SERVER_BUILD_INDEX_ERROR, "Engine failed: " + stat.ToString());
-        }
-
-        rc.ElapseFromBegin("totally cost");
-    } catch (std::exception& ex) {
-        SERVER_LOG_ERROR << "BuildIndexTask encounter exception: " << ex.what();
-        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
-    }
-
-    return SERVER_SUCCESS;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-HasTableTask::HasTableTask(const std::string& table_name, bool& has_table)
-    : BaseTask(DDL_DML_TASK_GROUP),
-      table_name_(table_name),
-      has_table_(has_table) {
-
-}
-
-BaseTaskPtr HasTableTask::Create(const std::string& table_name, bool& has_table) {
-    return std::shared_ptr<BaseTask>(new HasTableTask(table_name, has_table));
-}
-
-ServerError HasTableTask::OnExecute() {
-    try {
-        TimeRecorder rc("HasTableTask");
-
-        //step 1: check arguments
-        ServerError res = SERVER_SUCCESS;
-        res = ValidationUtil::ValidateTableName(table_name_);
-        if(res != SERVER_SUCCESS) {
-            return SetError(res, "Invalid table name: " + table_name_);
-        }
-        //step 2: check table existence
-        engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_);
-        if(!stat.ok()) {
-            return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
-        }
-
-        rc.ElapseFromBegin("totally cost");
-    } catch (std::exception& ex) {
-        SERVER_LOG_ERROR << "HasTableTask encounter exception: " << ex.what();
-        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
-    }
-
-    return SERVER_SUCCESS;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-DeleteTableTask::DeleteTableTask(const std::string& table_name)
-    : BaseTask(DDL_DML_TASK_GROUP),
-      table_name_(table_name) {
-
-}
-
-BaseTaskPtr DeleteTableTask::Create(const std::string& table_name) {
-    return std::shared_ptr<BaseTask>(new DeleteTableTask(table_name));
-}
-
-ServerError DeleteTableTask::OnExecute() {
-    try {
-        TimeRecorder rc("DeleteTableTask");
-
-        //step 1: check arguments
-        ServerError res = SERVER_SUCCESS;
-        res = ValidationUtil::ValidateTableName(table_name_);
-        if(res != SERVER_SUCCESS) {
-            return SetError(res, "Invalid table name: " + table_name_);
-        }
-
-        //step 2: check table existence
-        engine::meta::TableSchema table_info;
-        table_info.table_id_ = table_name_;
-        engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
-        if(!stat.ok()) {
-            if(stat.IsNotFound()) {
-                return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
-            } else {
-                return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
-            }
-        }
-
-        //step 3: delete table
-        std::vector<DB_DATE> dates;
-        stat = DBWrapper::DB()->DeleteTable(table_name_, dates);
-        if(!stat.ok()) {
-            return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
-        }
-
-        rc.ElapseFromBegin("totally cost");
-    } catch (std::exception& ex) {
-        SERVER_LOG_ERROR << "DeleteTableTask encounter exception: " << ex.what();
-        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
-    }
-
-    return SERVER_SUCCESS;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-ShowTablesTask::ShowTablesTask(std::vector<std::string>& tables)
-    : BaseTask(DDL_DML_TASK_GROUP),
-      tables_(tables) {
-
-}
-
-BaseTaskPtr ShowTablesTask::Create(std::vector<std::string>& tables) {
-    return std::shared_ptr<BaseTask>(new ShowTablesTask(tables));
-}
-
-ServerError ShowTablesTask::OnExecute() {
-    std::vector<engine::meta::TableSchema> schema_array;
-    engine::Status stat = DBWrapper::DB()->AllTables(schema_array);
-    if(!stat.ok()) {
-        return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
-    }
-
-    tables_.clear();
-    for(auto& schema : schema_array) {
-        tables_.push_back(schema.table_id_);
-    }
-
-    return SERVER_SUCCESS;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-AddVectorTask::AddVectorTask(const std::string& table_name,
-                                       const std::vector<thrift::RowRecord>& record_array,
-                                       std::vector<int64_t>& record_ids)
-    : BaseTask(DDL_DML_TASK_GROUP),
-      table_name_(table_name),
-      record_array_(record_array),
-      record_ids_(record_ids) {
-    record_ids_.clear();
-}
-
-BaseTaskPtr AddVectorTask::Create(const std::string& table_name,
-                                       const std::vector<thrift::RowRecord>& record_array,
-                                       std::vector<int64_t>& record_ids) {
-    return std::shared_ptr<BaseTask>(new AddVectorTask(table_name, record_array, record_ids));
-}
-
-ServerError AddVectorTask::OnExecute() {
-    try {
-        TimeRecorder rc("AddVectorTask");
-
-        //step 1: check arguments
-        ServerError res = SERVER_SUCCESS;
-        res = ValidationUtil::ValidateTableName(table_name_);
-        if(res != SERVER_SUCCESS) {
-            return SetError(res, "Invalid table name: " + table_name_);
-        }
-
-        if(record_array_.empty()) {
-            return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty");
-        }
-
-        //step 2: check table existence
-        engine::meta::TableSchema table_info;
-        table_info.table_id_ = table_name_;
-        engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
-        if(!stat.ok()) {
-            if(stat.IsNotFound()) {
-                return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
-            } else {
-                return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
-            }
-        }
-
-        rc.RecordSection("check validation");
-
-#ifdef MILVUS_ENABLE_PROFILING
-        std::string fname = "/tmp/insert_" + std::to_string(this->record_array_.size()) +
-                            "_" + GetCurrTimeStr() + ".profiling";
-        ProfilerStart(fname.c_str());
-#endif
-
-        //step 3: prepare float data
-        std::vector<float> vec_f;
-        ServerError error_code = SERVER_SUCCESS;
-        std::string error_msg;
-        ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg);
-        if(error_code != SERVER_SUCCESS) {
-            return SetError(error_code, error_msg);
-        }
-
-        rc.RecordSection("prepare vectors data");
-
-        //step 4: insert vectors
-        uint64_t vec_count = (uint64_t)record_array_.size();
-        stat = DBWrapper::DB()->InsertVectors(table_name_, vec_count, vec_f.data(), record_ids_);
-        if(!stat.ok()) {
-            return SetError(SERVER_CACHE_ERROR, "Cache error: " + stat.ToString());
-        }
-
-        if(record_ids_.size() != vec_count) {
-            std::string msg = "Add " + std::to_string(vec_count) + " vectors but only return "
-                    + std::to_string(record_ids_.size()) + " id";
-            return SetError(SERVER_ILLEGAL_VECTOR_ID, msg);
-        }
-
-#ifdef MILVUS_ENABLE_PROFILING
-        ProfilerStop();
-#endif
-
-        rc.RecordSection("add vectors to engine");
-        rc.ElapseFromBegin("totally cost");
-
-    } catch (std::exception& ex) {
-        SERVER_LOG_ERROR << "AddVectorTask encounter exception: " << ex.what();
-        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
-    }
-
-    return SERVER_SUCCESS;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-SearchVectorTaskBase::SearchVectorTaskBase(const std::string &table_name,
-        const std::vector<std::string>& file_id_array,
-        const std::vector<thrift::RowRecord> &query_record_array,
-        const std::vector<thrift::Range> &query_range_array,
-        const int64_t top_k)
-    : BaseTask(DQL_TASK_GROUP),
-      table_name_(table_name),
-      file_id_array_(file_id_array),
-      record_array_(query_record_array),
-      range_array_(query_range_array),
-      top_k_(top_k) {
-
-}
-
-ServerError SearchVectorTaskBase::OnExecute() {
-    try {
-        std::string title = "SearchVectorTask(n=" + std::to_string(record_array_.size())
-                + " k=" + std::to_string(top_k_) + ")";
-        TimeRecorder rc(title);
-
-        //step 1: check arguments
-        ServerError res = SERVER_SUCCESS;
-        res = ValidationUtil::ValidateTableName(table_name_);
-        if(res != SERVER_SUCCESS) {
-            return SetError(res, "Invalid table name: " + table_name_);
-        }
-
-        if(top_k_ <= 0 || top_k_ > 1024) {
-            return SetError(SERVER_INVALID_TOPK, "Invalid topk: " + std::to_string(top_k_));
-        }
-        if(record_array_.empty()) {
-            return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty");
-        }
-
-        //step 2: check table existence
-        engine::meta::TableSchema table_info;
-        table_info.table_id_ = table_name_;
-        engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
-        if(!stat.ok()) {
-            if(stat.IsNotFound()) {
-                return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
-            } else {
-                return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
-            }
-        }
-
-        //step 3: check date range, and convert to db dates
-        std::vector<DB_DATE> dates;
-        ServerError error_code = SERVER_SUCCESS;
-        std::string error_msg;
-        ConvertTimeRangeToDBDates(range_array_, dates, error_code, error_msg);
-        if(error_code != SERVER_SUCCESS) {
-            return SetError(error_code, error_msg);
-        }
-
-        double span_check = rc.RecordSection("check validation");
-
-#ifdef MILVUS_ENABLE_PROFILING
-        std::string fname = "/tmp/search_nq_" + std::to_string(this->record_array_.size()) +
-                            "_top_" + std::to_string(this->top_k_) + "_" +
-                            GetCurrTimeStr() + ".profiling";
-        ProfilerStart(fname.c_str());
-#endif
-
-        //step 3: prepare float data
-        std::vector<float> vec_f;
-        ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg);
-        if(error_code != SERVER_SUCCESS) {
-            return SetError(error_code, error_msg);
-        }
-
-        double span_prepare = rc.RecordSection("prepare vector data");
-
-        //step 4: search vectors
-        engine::QueryResults results;
-        uint64_t record_count = (uint64_t)record_array_.size();
-
-        if(file_id_array_.empty()) {
-            stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k_, record_count, vec_f.data(), dates, results);
-        } else {
-            stat = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t) top_k_, record_count, vec_f.data(), dates, results);
-        }
-
-        double span_search = rc.RecordSection("search vectors from engine");
-        if(!stat.ok()) {
-            return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
-        }
-
-        if(results.empty()) {
-            return SERVER_SUCCESS; //empty table
-        }
-
-        if(results.size() != record_count) {
-            std::string msg = "Search " + std::to_string(record_count) + " vectors but only return "
-                              + std::to_string(results.size()) + " results";
-            return SetError(SERVER_ILLEGAL_SEARCH_RESULT, msg);
-        }
-
-        //step 5: construct result array
-        ConstructResult(results);
-
-#ifdef MILVUS_ENABLE_PROFILING
-        ProfilerStop();
-#endif
-
-        double span_result = rc.RecordSection("construct result");
-        rc.ElapseFromBegin("totally cost");
-
-        //step 6: print time cost percent
-        double total_cost = span_check + span_prepare + span_search + span_result;
-        SERVER_LOG_DEBUG << title << ": check validation(" << (span_check/total_cost)*100.0 << "%)"
-            << " prepare data(" << (span_prepare/total_cost)*100.0 << "%)"
-            << " search(" << (span_search/total_cost)*100.0 << "%)"
-            << " construct result(" << (span_result/total_cost)*100.0 << "%)";
-
-    } catch (std::exception& ex) {
-        SERVER_LOG_ERROR << "SearchVectorTask encounter exception: " << ex.what();
-        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
-    }
-
-    return SERVER_SUCCESS;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-SearchVectorTask1::SearchVectorTask1(const std::string &table_name,
-                                     const std::vector<std::string>& file_id_array,
-                                     const std::vector<thrift::RowRecord> &query_record_array,
-                                     const std::vector<thrift::Range> &query_range_array,
-                                     const int64_t top_k,
-                                     std::vector<thrift::TopKQueryResult> &result_array)
-        : SearchVectorTaskBase(table_name, file_id_array, query_record_array, query_range_array, top_k),
-          result_array_(result_array) {
-
-}
-
-BaseTaskPtr SearchVectorTask1::Create(const std::string& table_name,
-                                      const std::vector<std::string>& file_id_array,
-                                      const std::vector<thrift::RowRecord> & query_record_array,
-                                      const std::vector<thrift::Range> & query_range_array,
-                                      const int64_t top_k,
-                                      std::vector<thrift::TopKQueryResult>& result_array) {
-    return std::shared_ptr<BaseTask>(new SearchVectorTask1(table_name, file_id_array,
-                                                           query_record_array, query_range_array, top_k, result_array));
-}
-
-ServerError SearchVectorTask1::ConstructResult(engine::QueryResults& results) {
-    for(uint64_t i = 0; i < results.size(); i++) {
-        auto& result = results[i];
-        const auto& record = record_array_[i];
-
-        thrift::TopKQueryResult thrift_topk_result;
-        for(auto& pair : result) {
-            thrift::QueryResult thrift_result;
-            thrift_result.__set_id(pair.first);
-            thrift_result.__set_distance(pair.second);
-
-            thrift_topk_result.query_result_arrays.emplace_back(thrift_result);
-        }
-
-        result_array_.emplace_back(thrift_topk_result);
-    }
-
-    return SERVER_SUCCESS;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-SearchVectorTask2::SearchVectorTask2(const std::string &table_name,
-                                     const std::vector<std::string>& file_id_array,
-                                     const std::vector<thrift::RowRecord> &query_record_array,
-                                     const std::vector<thrift::Range> &query_range_array,
-                                     const int64_t top_k,
-                                     std::vector<thrift::TopKQueryBinResult> &result_array)
-    : SearchVectorTaskBase(table_name, file_id_array, query_record_array, query_range_array, top_k),
-      result_array_(result_array) {
-
-}
-
-BaseTaskPtr SearchVectorTask2::Create(const std::string& table_name,
-                                     const std::vector<std::string>& file_id_array,
-                                     const std::vector<thrift::RowRecord> & query_record_array,
-                                     const std::vector<thrift::Range> & query_range_array,
-                                     const int64_t top_k,
-                                     std::vector<thrift::TopKQueryBinResult>& result_array) {
-    return std::shared_ptr<BaseTask>(new SearchVectorTask2(table_name, file_id_array,
-            query_record_array, query_range_array, top_k, result_array));
-}
-
-ServerError SearchVectorTask2::ConstructResult(engine::QueryResults& results) {
-    for(size_t i = 0; i < results.size(); i++) {
-        auto& result = results[i];
-
-        thrift::TopKQueryBinResult thrift_topk_result;
-        if(result.empty()) {
-            result_array_.emplace_back(thrift_topk_result);
-            continue;
-        }
-
-        std::string str_ids, str_distances;
-        str_ids.resize(sizeof(engine::IDNumber)*result.size());
-        str_distances.resize(sizeof(double)*result.size());
-
-        engine::IDNumber* ids_ptr = (engine::IDNumber*)str_ids.data();
-        double* distance_ptr = (double*)str_distances.data();
-        for(size_t k = 0; k < result.size(); k++) {
-            auto& pair = result[k];
-            ids_ptr[k] = pair.first;
-            distance_ptr[k] = pair.second;
-        }
-
-        thrift_topk_result.__set_id_array(str_ids);
-        thrift_topk_result.__set_distance_array(str_distances);
-        result_array_.emplace_back(thrift_topk_result);
-    }
-
-    return SERVER_SUCCESS;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-GetTableRowCountTask::GetTableRowCountTask(const std::string& table_name, int64_t& row_count)
-: BaseTask(DDL_DML_TASK_GROUP),
-  table_name_(table_name),
-  row_count_(row_count) {
-
-}
-
-BaseTaskPtr GetTableRowCountTask::Create(const std::string& table_name, int64_t& row_count) {
-    return std::shared_ptr<BaseTask>(new GetTableRowCountTask(table_name, row_count));
-}
-
-ServerError GetTableRowCountTask::OnExecute() {
-    try {
-        TimeRecorder rc("GetTableRowCountTask");
-
-        //step 1: check arguments
-        ServerError res = SERVER_SUCCESS;
-        res = ValidationUtil::ValidateTableName(table_name_);
-        if(res != SERVER_SUCCESS) {
-            return SetError(res, "Invalid table name: " + table_name_);
-        }
-
-        //step 2: get row count
-        uint64_t row_count = 0;
-        engine::Status stat = DBWrapper::DB()->GetTableRowCount(table_name_, row_count);
-        if (!stat.ok()) {
-            return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString());
-        }
-
-        row_count_ = (int64_t) row_count;
-
-        rc.ElapseFromBegin("totally cost");
-
-    } catch (std::exception& ex) {
-        SERVER_LOG_ERROR << "GetTableRowCountTask encounter exception: " << ex.what();
-        return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
-    }
-
-    return SERVER_SUCCESS;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-PingTask::PingTask(const std::string& cmd, std::string& result)
-    : BaseTask(PING_TASK_GROUP),
-      cmd_(cmd),
-      result_(result) {
-
-}
-
-BaseTaskPtr PingTask::Create(const std::string& cmd, std::string& result) {
-    return std::shared_ptr<BaseTask>(new PingTask(cmd, result));
-}
-
-ServerError PingTask::OnExecute() {
-    if(cmd_ == "version") {
-        result_ = MILVUS_VERSION;
-    }
-
-    return SERVER_SUCCESS;
-}
-
-}
-}
-}

cpp/src/server/thrift_impl/RequestTask.h

@@ -1,232 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#pragma once
-
-#include "RequestScheduler.h"
-#include "utils/Error.h"
-#include "db/Types.h"
-
-#include "milvus_types.h"
-
-#include <condition_variable>
-#include <memory>
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class CreateTableTask : public BaseTask {
-public:
-    static BaseTaskPtr Create(const ::milvus::thrift::TableSchema& schema);
-
-protected:
-    CreateTableTask(const ::milvus::thrift::TableSchema& schema);
-
-    ServerError OnExecute() override;
-
-private:
-    const ::milvus::thrift::TableSchema& schema_;
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class HasTableTask : public BaseTask {
-public:
-    static BaseTaskPtr Create(const std::string& table_name, bool& has_table);
-
-protected:
-    HasTableTask(const std::string& table_name, bool& has_table);
-
-    ServerError OnExecute() override;
-
-
-private:
-    std::string table_name_;
-    bool& has_table_;
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class DescribeTableTask : public BaseTask {
-public:
-    static BaseTaskPtr Create(const std::string& table_name, ::milvus::thrift::TableSchema& schema);
-
-protected:
-    DescribeTableTask(const std::string& table_name, ::milvus::thrift::TableSchema& schema);
-
-    ServerError OnExecute() override;
-
-
-private:
-    std::string table_name_;
-    ::milvus::thrift::TableSchema& schema_;
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class DeleteTableTask : public BaseTask {
-public:
-    static BaseTaskPtr Create(const std::string& table_name);
-
-protected:
-    DeleteTableTask(const std::string& table_name);
-
-    ServerError OnExecute() override;
-
-
-private:
-    std::string table_name_;
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class BuildIndexTask : public BaseTask {
-public:
-    static BaseTaskPtr Create(const std::string& table_name);
-
-protected:
-    BuildIndexTask(const std::string& table_name);
-
-    ServerError OnExecute() override;
-
-
-private:
-    std::string table_name_;
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class ShowTablesTask : public BaseTask {
-public:
-    static BaseTaskPtr Create(std::vector<std::string>& tables);
-
-protected:
-    ShowTablesTask(std::vector<std::string>& tables);
-
-    ServerError OnExecute() override;
-
-private:
-    std::vector<std::string>& tables_;
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class AddVectorTask : public BaseTask {
-public:
-    static BaseTaskPtr Create(const std::string& table_name,
-                              const std::vector<::milvus::thrift::RowRecord>& record_array,
-                              std::vector<int64_t>& record_ids_);
-
-protected:
-    AddVectorTask(const std::string& table_name,
-                        const std::vector<::milvus::thrift::RowRecord>& record_array,
-                        std::vector<int64_t>& record_ids_);
-
-    ServerError OnExecute() override;
-
-private:
-    std::string table_name_;
-    const std::vector<::milvus::thrift::RowRecord>& record_array_;
-    std::vector<int64_t>& record_ids_;
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class SearchVectorTaskBase : public BaseTask {
-protected:
-    SearchVectorTaskBase(const std::string& table_name,
-                        const std::vector<std::string>& file_id_array,
-                        const std::vector<::milvus::thrift::RowRecord> & query_record_array,
-                        const std::vector<::milvus::thrift::Range> & query_range_array,
-                        const int64_t top_k);
-
-    ServerError OnExecute() override;
-
-    virtual ServerError ConstructResult(engine::QueryResults& results) = 0;
-
-protected:
-    std::string table_name_;
-    std::vector<std::string> file_id_array_;
-    int64_t top_k_;
-    const std::vector<::milvus::thrift::RowRecord>& record_array_;
-    const std::vector<::milvus::thrift::Range>& range_array_;
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class SearchVectorTask1 : public SearchVectorTaskBase {
-public:
-    static BaseTaskPtr Create(const std::string& table_name,
-                              const std::vector<std::string>& file_id_array,
-                              const std::vector<::milvus::thrift::RowRecord> & query_record_array,
-                              const std::vector<::milvus::thrift::Range> & query_range_array,
-                              const int64_t top_k,
-                              std::vector<::milvus::thrift::TopKQueryResult>& result_array);
-
-protected:
-    SearchVectorTask1(const std::string& table_name,
-                      const std::vector<std::string>& file_id_array,
-                      const std::vector<::milvus::thrift::RowRecord> & query_record_array,
-                      const std::vector<::milvus::thrift::Range> & query_range_array,
-                      const int64_t top_k,
-                     std::vector<::milvus::thrift::TopKQueryResult>& result_array);
-
-    ServerError ConstructResult(engine::QueryResults& results) override;
-
-private:
-    std::vector<::milvus::thrift::TopKQueryResult>& result_array_;
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class SearchVectorTask2 : public SearchVectorTaskBase {
-public:
-    static BaseTaskPtr Create(const std::string& table_name,
-                              const std::vector<std::string>& file_id_array,
-                              const std::vector<::milvus::thrift::RowRecord> & query_record_array,
-                              const std::vector<::milvus::thrift::Range> & query_range_array,
-                              const int64_t top_k,
-                              std::vector<::milvus::thrift::TopKQueryBinResult>& result_array);
-
-protected:
-    SearchVectorTask2(const std::string& table_name,
-                      const std::vector<std::string>& file_id_array,
-                      const std::vector<::milvus::thrift::RowRecord> & query_record_array,
-                      const std::vector<::milvus::thrift::Range> & query_range_array,
-                      const int64_t top_k,
-                      std::vector<::milvus::thrift::TopKQueryBinResult>& result_array);
-
-    ServerError ConstructResult(engine::QueryResults& results) override;
-
-private:
-    std::vector<::milvus::thrift::TopKQueryBinResult>& result_array_;
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class GetTableRowCountTask : public BaseTask {
-public:
-    static BaseTaskPtr Create(const std::string& table_name, int64_t& row_count);
-
-protected:
-    GetTableRowCountTask(const std::string& table_name, int64_t& row_count);
-
-    ServerError OnExecute() override;
-
-private:
-    std::string table_name_;
-    int64_t& row_count_;
-};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class PingTask : public BaseTask {
-public:
-    static BaseTaskPtr Create(const std::string& cmd, std::string& result);
-
-protected:
-    PingTask(const std::string& cmd, std::string& result);
-
-    ServerError OnExecute() override;
-
-private:
-    std::string cmd_;
-    std::string& result_;
-};
-
-}
-}
-}

cpp/src/server/thrift_impl/ThreadPoolServer.cpp

@@ -1,36 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#include "metrics/Metrics.h"
-#include "ThreadPoolServer.h"
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-void
-ThreadPoolServer::onClientConnected(const std::shared_ptr<apache::thrift::server::TConnectedClient> &pClient) {
-    server::Metrics::GetInstance().ConnectionGaugeIncrement();
-    TThreadPoolServer::onClientConnected(pClient);
-}
-
-void
-ThreadPoolServer::onClientDisconnected(apache::thrift::server::TConnectedClient *pClient) {
-    server::Metrics::GetInstance().ConnectionGaugeDecrement();
-    TThreadPoolServer::onClientDisconnected(pClient);
-}
-
-zilliz::milvus::server::ThreadPoolServer::ThreadPoolServer(const std::shared_ptr<apache::thrift::TProcessor> &processor,
-                                                           const std::shared_ptr<apache::thrift::transport::TServerTransport> &serverTransport,
-                                                           const std::shared_ptr<apache::thrift::transport::TTransportFactory> &transportFactory,
-                                                           const std::shared_ptr<apache::thrift::protocol::TProtocolFactory> &protocolFactory,
-                                                           const std::shared_ptr<apache::thrift::concurrency::ThreadManager> &threadManager)
-        : TThreadPoolServer(processor, serverTransport, transportFactory, protocolFactory, threadManager) {
-
-}
-
-}
-}
-}

cpp/src/server/thrift_impl/ThreadPoolServer.h

@@ -1,33 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-
-#pragma once
-
-#include <thrift/server/TThreadPoolServer.h>
-
-
-namespace zilliz {
-namespace milvus {
-namespace server {
-
-class ThreadPoolServer : public apache::thrift::server::TThreadPoolServer {
- public:
-    ThreadPoolServer(
-        const std::shared_ptr<apache::thrift::TProcessor>& processor,
-        const std::shared_ptr<apache::thrift::transport::TServerTransport>& serverTransport,
-        const std::shared_ptr<apache::thrift::transport::TTransportFactory>& transportFactory,
-        const std::shared_ptr<apache::thrift::protocol::TProtocolFactory>& protocolFactory,
-        const std::shared_ptr<apache::thrift::concurrency::ThreadManager>& threadManager
-        = apache::thrift::concurrency::ThreadManager::newSimpleThreadManager());
-
- protected:
-    void onClientConnected(const std::shared_ptr<apache::thrift::server::TConnectedClient>& pClient) override ;
-    void onClientDisconnected(apache::thrift::server::TConnectedClient* pClient) override ;
-};
-
-}
-}
-}

cpp/src/thrift/cpp_gen.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-thrift -r --gen cpp ./milvus.thrift
-

cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp

@@ -1,241 +0,0 @@
-// This autogenerated skeleton file illustrates how to build a server.
-// You should copy it to another filename to avoid overwriting it.
-
-#include "MilvusService.h"
-#include <thrift/protocol/TBinaryProtocol.h>
-#include <thrift/server/TSimpleServer.h>
-#include <thrift/transport/TServerSocket.h>
-#include <thrift/transport/TBufferTransports.h>
-
-using namespace ::apache::thrift;
-using namespace ::apache::thrift::protocol;
-using namespace ::apache::thrift::transport;
-using namespace ::apache::thrift::server;
-
-using namespace  ::milvus::thrift;
-
-class MilvusServiceHandler : virtual public MilvusServiceIf {
- public:
-  MilvusServiceHandler() {
-    // Your initialization goes here
-  }
-
-  /**
-   * @brief Create table method
-   * 
-   * This method is used to create table
-   * 
-   * @param param, use to provide table information to be created.
-   * 
-   * 
-   * @param param
-   */
-  void CreateTable(const TableSchema& param) {
-    // Your implementation goes here
-    printf("CreateTable\n");
-  }
-
-  /**
-   * @brief Test table existence method
-   * 
-   * This method is used to test table existence.
-   * 
-   * @param table_name, table name is going to be tested.
-   * 
-   * 
-   * @param table_name
-   */
-  bool HasTable(const std::string& table_name) {
-    // Your implementation goes here
-    printf("HasTable\n");
-  }
-
-  /**
-   * @brief Delete table method
-   * 
-   * This method is used to delete table.
-   * 
-   * @param table_name, table name is going to be deleted.
-   * 
-   * 
-   * @param table_name
-   */
-  void DeleteTable(const std::string& table_name) {
-    // Your implementation goes here
-    printf("DeleteTable\n");
-  }
-
-  /**
-   * @brief Build index by table method
-   * 
-   * This method is used to build index by table in sync mode.
-   * 
-   * @param table_name, table is going to be built index.
-   * 
-   * 
-   * @param table_name
-   */
-  void BuildIndex(const std::string& table_name) {
-    // Your implementation goes here
-    printf("BuildIndex\n");
-  }
-
-  /**
-   * @brief Add vector array to table
-   * 
-   * This method is used to add vector array to table.
-   * 
-   * @param table_name, table_name is inserted.
-   * @param record_array, vector array is inserted.
-   * 
-   * @return vector id array
-   * 
-   * @param table_name
-   * @param record_array
-   */
-  void AddVector(std::vector<int64_t> & _return, const std::string& table_name, const std::vector<RowRecord> & record_array) {
-    // Your implementation goes here
-    printf("AddVector\n");
-  }
-
-  /**
-   * @brief Query vector
-   * 
-   * This method is used to query vector in table.
-   * 
-   * @param table_name, table_name is queried.
-   * @param query_record_array, all vector are going to be queried.
-   * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-   * @param topk, how many similarity vectors will be searched.
-   * 
-   * @return query result array.
-   * 
-   * @param table_name
-   * @param query_record_array
-   * @param query_range_array
-   * @param topk
-   */
-  void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk) {
-    // Your implementation goes here
-    printf("SearchVector\n");
-  }
-
-  /**
-   * @brief Query vector
-   * 
-   * This method is used to query vector in table.
-   * 
-   * @param table_name, table_name is queried.
-   * @param query_record_array, all vector are going to be queried.
-   * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-   * @param topk, how many similarity vectors will be searched.
-   * 
-   * @return query binary result array.
-   * 
-   * @param table_name
-   * @param query_record_array
-   * @param query_range_array
-   * @param topk
-   */
-  void SearchVector2(std::vector<TopKQueryBinResult> & _return, const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk) {
-    // Your implementation goes here
-    printf("SearchVector2\n");
-  }
-
-  /**
-   * @brief Internal use query interface
-   * 
-   * This method is used to query vector in specified files.
-   * 
-   * @param file_id_array, specified files id array, queried.
-   * @param query_record_array, all vector are going to be queried.
-   * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-   * @param topk, how many similarity vectors will be searched.
-   * 
-   * @return query result array.
-   * 
-   * @param table_name
-   * @param file_id_array
-   * @param query_record_array
-   * @param query_range_array
-   * @param topk
-   */
-  void SearchVectorInFiles(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<std::string> & file_id_array, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk) {
-    // Your implementation goes here
-    printf("SearchVectorInFiles\n");
-  }
-
-  /**
-   * @brief Get table schema
-   * 
-   * This method is used to get table schema.
-   * 
-   * @param table_name, target table name.
-   * 
-   * @return table schema
-   * 
-   * @param table_name
-   */
-  void DescribeTable(TableSchema& _return, const std::string& table_name) {
-    // Your implementation goes here
-    printf("DescribeTable\n");
-  }
-
-  /**
-   * @brief Get table row count
-   * 
-   * This method is used to get table row count.
-   * 
-   * @param table_name, target table name.
-   * 
-   * @return table row count
-   * 
-   * @param table_name
-   */
-  int64_t GetTableRowCount(const std::string& table_name) {
-    // Your implementation goes here
-    printf("GetTableRowCount\n");
-  }
-
-  /**
-   * @brief List all tables in database
-   * 
-   * This method is used to list all tables.
-   * 
-   * 
-   * @return table names.
-   */
-  void ShowTables(std::vector<std::string> & _return) {
-    // Your implementation goes here
-    printf("ShowTables\n");
-  }
-
-  /**
-   * @brief Give the server status
-   * 
-   * This method is used to give the server status.
-   * 
-   * @return Server status.
-   * 
-   * @param cmd
-   */
-  void Ping(std::string& _return, const std::string& cmd) {
-    // Your implementation goes here
-    printf("Ping\n");
-  }
-
-};
-
-int main(int argc, char **argv) {
-  int port = 9090;
-  ::apache::thrift::stdcxx::shared_ptr<MilvusServiceHandler> handler(new MilvusServiceHandler());
-  ::apache::thrift::stdcxx::shared_ptr<TProcessor> processor(new MilvusServiceProcessor(handler));
-  ::apache::thrift::stdcxx::shared_ptr<TServerTransport> serverTransport(new TServerSocket(port));
-  ::apache::thrift::stdcxx::shared_ptr<TTransportFactory> transportFactory(new TBufferedTransportFactory());
-  ::apache::thrift::stdcxx::shared_ptr<TProtocolFactory> protocolFactory(new TBinaryProtocolFactory());
-
-  TSimpleServer server(processor, serverTransport, transportFactory, protocolFactory);
-  server.serve();
-  return 0;
-}
-

cpp/src/thrift/gen-cpp/milvus_constants.cpp

@@ -1,17 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.12.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-#include "milvus_constants.h"
-
-namespace milvus { namespace thrift {
-
-const milvusConstants g_milvus_constants;
-
-milvusConstants::milvusConstants() {
-}
-
-}} // namespace
-

cpp/src/thrift/gen-cpp/milvus_constants.h

@@ -1,24 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.12.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-#ifndef milvus_CONSTANTS_H
-#define milvus_CONSTANTS_H
-
-#include "milvus_types.h"
-
-namespace milvus { namespace thrift {
-
-class milvusConstants {
- public:
-  milvusConstants();
-
-};
-
-extern const milvusConstants g_milvus_constants;
-
-}} // namespace
-
-#endif

cpp/src/thrift/gen-cpp/milvus_types.cpp

@@ -1,899 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.12.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-#include "milvus_types.h"
-
-#include <algorithm>
-#include <ostream>
-
-#include <thrift/TToString.h>
-
-namespace milvus { namespace thrift {
-
-int _kErrorCodeValues[] = {
-  ErrorCode::SUCCESS,
-  ErrorCode::UNEXPECTED_ERROR,
-  ErrorCode::CONNECT_FAILED,
-  ErrorCode::PERMISSION_DENIED,
-  ErrorCode::TABLE_NOT_EXISTS,
-  ErrorCode::ILLEGAL_ARGUMENT,
-  ErrorCode::ILLEGAL_RANGE,
-  ErrorCode::ILLEGAL_DIMENSION,
-  ErrorCode::ILLEGAL_INDEX_TYPE,
-  ErrorCode::ILLEGAL_TABLE_NAME,
-  ErrorCode::ILLEGAL_TOPK,
-  ErrorCode::ILLEGAL_ROWRECORD,
-  ErrorCode::ILLEGAL_VECTOR_ID,
-  ErrorCode::ILLEGAL_SEARCH_RESULT,
-  ErrorCode::FILE_NOT_FOUND,
-  ErrorCode::META_FAILED,
-  ErrorCode::CACHE_FAILED,
-  ErrorCode::CANNOT_CREATE_FOLDER,
-  ErrorCode::CANNOT_CREATE_FILE,
-  ErrorCode::CANNOT_DELETE_FOLDER,
-  ErrorCode::CANNOT_DELETE_FILE,
-  ErrorCode::BUILD_INDEX_ERROR
-};
-const char* _kErrorCodeNames[] = {
-  "SUCCESS",
-  "UNEXPECTED_ERROR",
-  "CONNECT_FAILED",
-  "PERMISSION_DENIED",
-  "TABLE_NOT_EXISTS",
-  "ILLEGAL_ARGUMENT",
-  "ILLEGAL_RANGE",
-  "ILLEGAL_DIMENSION",
-  "ILLEGAL_INDEX_TYPE",
-  "ILLEGAL_TABLE_NAME",
-  "ILLEGAL_TOPK",
-  "ILLEGAL_ROWRECORD",
-  "ILLEGAL_VECTOR_ID",
-  "ILLEGAL_SEARCH_RESULT",
-  "FILE_NOT_FOUND",
-  "META_FAILED",
-  "CACHE_FAILED",
-  "CANNOT_CREATE_FOLDER",
-  "CANNOT_CREATE_FILE",
-  "CANNOT_DELETE_FOLDER",
-  "CANNOT_DELETE_FILE",
-  "BUILD_INDEX_ERROR"
-};
-const std::map<int, const char*> _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(22, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
-
-std::ostream& operator<<(std::ostream& out, const ErrorCode::type& val) {
-  std::map<int, const char*>::const_iterator it = _ErrorCode_VALUES_TO_NAMES.find(val);
-  if (it != _ErrorCode_VALUES_TO_NAMES.end()) {
-    out << it->second;
-  } else {
-    out << static_cast<int>(val);
-  }
-  return out;
-}
-
-
-Exception::~Exception() throw() {
-}
-
-
-void Exception::__set_code(const ErrorCode::type val) {
-  this->code = val;
-}
-
-void Exception::__set_reason(const std::string& val) {
-  this->reason = val;
-}
-std::ostream& operator<<(std::ostream& out, const Exception& obj)
-{
-  obj.printTo(out);
-  return out;
-}
-
-
-uint32_t Exception::read(::apache::thrift::protocol::TProtocol* iprot) {
-
-  ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
-  uint32_t xfer = 0;
-  std::string fname;
-  ::apache::thrift::protocol::TType ftype;
-  int16_t fid;
-
-  xfer += iprot->readStructBegin(fname);
-
-  using ::apache::thrift::protocol::TProtocolException;
-
-
-  while (true)
-  {
-    xfer += iprot->readFieldBegin(fname, ftype, fid);
-    if (ftype == ::apache::thrift::protocol::T_STOP) {
-      break;
-    }
-    switch (fid)
-    {
-      case 1:
-        if (ftype == ::apache::thrift::protocol::T_I32) {
-          int32_t ecast0;
-          xfer += iprot->readI32(ecast0);
-          this->code = (ErrorCode::type)ecast0;
-          this->__isset.code = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      case 2:
-        if (ftype == ::apache::thrift::protocol::T_STRING) {
-          xfer += iprot->readString(this->reason);
-          this->__isset.reason = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      default:
-        xfer += iprot->skip(ftype);
-        break;
-    }
-    xfer += iprot->readFieldEnd();
-  }
-
-  xfer += iprot->readStructEnd();
-
-  return xfer;
-}
-
-uint32_t Exception::write(::apache::thrift::protocol::TProtocol* oprot) const {
-  uint32_t xfer = 0;
-  ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
-  xfer += oprot->writeStructBegin("Exception");
-
-  xfer += oprot->writeFieldBegin("code", ::apache::thrift::protocol::T_I32, 1);
-  xfer += oprot->writeI32((int32_t)this->code);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldBegin("reason", ::apache::thrift::protocol::T_STRING, 2);
-  xfer += oprot->writeString(this->reason);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldStop();
-  xfer += oprot->writeStructEnd();
-  return xfer;
-}
-
-void swap(Exception &a, Exception &b) {
-  using ::std::swap;
-  swap(a.code, b.code);
-  swap(a.reason, b.reason);
-  swap(a.__isset, b.__isset);
-}
-
-Exception::Exception(const Exception& other1) : TException() {
-  code = other1.code;
-  reason = other1.reason;
-  __isset = other1.__isset;
-}
-Exception& Exception::operator=(const Exception& other2) {
-  code = other2.code;
-  reason = other2.reason;
-  __isset = other2.__isset;
-  return *this;
-}
-void Exception::printTo(std::ostream& out) const {
-  using ::apache::thrift::to_string;
-  out << "Exception(";
-  out << "code=" << to_string(code);
-  out << ", " << "reason=" << to_string(reason);
-  out << ")";
-}
-
-const char* Exception::what() const throw() {
-  try {
-    std::stringstream ss;
-    ss << "TException - service has thrown: " << *this;
-    this->thriftTExceptionMessageHolder_ = ss.str();
-    return this->thriftTExceptionMessageHolder_.c_str();
-  } catch (const std::exception&) {
-    return "TException - service has thrown: Exception";
-  }
-}
-
-
-TableSchema::~TableSchema() throw() {
-}
-
-
-void TableSchema::__set_table_name(const std::string& val) {
-  this->table_name = val;
-}
-
-void TableSchema::__set_index_type(const int32_t val) {
-  this->index_type = val;
-}
-
-void TableSchema::__set_dimension(const int64_t val) {
-  this->dimension = val;
-}
-
-void TableSchema::__set_store_raw_vector(const bool val) {
-  this->store_raw_vector = val;
-}
-std::ostream& operator<<(std::ostream& out, const TableSchema& obj)
-{
-  obj.printTo(out);
-  return out;
-}
-
-
-uint32_t TableSchema::read(::apache::thrift::protocol::TProtocol* iprot) {
-
-  ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
-  uint32_t xfer = 0;
-  std::string fname;
-  ::apache::thrift::protocol::TType ftype;
-  int16_t fid;
-
-  xfer += iprot->readStructBegin(fname);
-
-  using ::apache::thrift::protocol::TProtocolException;
-
-  bool isset_table_name = false;
-
-  while (true)
-  {
-    xfer += iprot->readFieldBegin(fname, ftype, fid);
-    if (ftype == ::apache::thrift::protocol::T_STOP) {
-      break;
-    }
-    switch (fid)
-    {
-      case 1:
-        if (ftype == ::apache::thrift::protocol::T_STRING) {
-          xfer += iprot->readString(this->table_name);
-          isset_table_name = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      case 2:
-        if (ftype == ::apache::thrift::protocol::T_I32) {
-          xfer += iprot->readI32(this->index_type);
-          this->__isset.index_type = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      case 3:
-        if (ftype == ::apache::thrift::protocol::T_I64) {
-          xfer += iprot->readI64(this->dimension);
-          this->__isset.dimension = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      case 4:
-        if (ftype == ::apache::thrift::protocol::T_BOOL) {
-          xfer += iprot->readBool(this->store_raw_vector);
-          this->__isset.store_raw_vector = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      default:
-        xfer += iprot->skip(ftype);
-        break;
-    }
-    xfer += iprot->readFieldEnd();
-  }
-
-  xfer += iprot->readStructEnd();
-
-  if (!isset_table_name)
-    throw TProtocolException(TProtocolException::INVALID_DATA);
-  return xfer;
-}
-
-uint32_t TableSchema::write(::apache::thrift::protocol::TProtocol* oprot) const {
-  uint32_t xfer = 0;
-  ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
-  xfer += oprot->writeStructBegin("TableSchema");
-
-  xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 1);
-  xfer += oprot->writeString(this->table_name);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldBegin("index_type", ::apache::thrift::protocol::T_I32, 2);
-  xfer += oprot->writeI32(this->index_type);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldBegin("dimension", ::apache::thrift::protocol::T_I64, 3);
-  xfer += oprot->writeI64(this->dimension);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldBegin("store_raw_vector", ::apache::thrift::protocol::T_BOOL, 4);
-  xfer += oprot->writeBool(this->store_raw_vector);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldStop();
-  xfer += oprot->writeStructEnd();
-  return xfer;
-}
-
-void swap(TableSchema &a, TableSchema &b) {
-  using ::std::swap;
-  swap(a.table_name, b.table_name);
-  swap(a.index_type, b.index_type);
-  swap(a.dimension, b.dimension);
-  swap(a.store_raw_vector, b.store_raw_vector);
-  swap(a.__isset, b.__isset);
-}
-
-TableSchema::TableSchema(const TableSchema& other3) {
-  table_name = other3.table_name;
-  index_type = other3.index_type;
-  dimension = other3.dimension;
-  store_raw_vector = other3.store_raw_vector;
-  __isset = other3.__isset;
-}
-TableSchema& TableSchema::operator=(const TableSchema& other4) {
-  table_name = other4.table_name;
-  index_type = other4.index_type;
-  dimension = other4.dimension;
-  store_raw_vector = other4.store_raw_vector;
-  __isset = other4.__isset;
-  return *this;
-}
-void TableSchema::printTo(std::ostream& out) const {
-  using ::apache::thrift::to_string;
-  out << "TableSchema(";
-  out << "table_name=" << to_string(table_name);
-  out << ", " << "index_type=" << to_string(index_type);
-  out << ", " << "dimension=" << to_string(dimension);
-  out << ", " << "store_raw_vector=" << to_string(store_raw_vector);
-  out << ")";
-}
-
-
-Range::~Range() throw() {
-}
-
-
-void Range::__set_start_value(const std::string& val) {
-  this->start_value = val;
-}
-
-void Range::__set_end_value(const std::string& val) {
-  this->end_value = val;
-}
-std::ostream& operator<<(std::ostream& out, const Range& obj)
-{
-  obj.printTo(out);
-  return out;
-}
-
-
-uint32_t Range::read(::apache::thrift::protocol::TProtocol* iprot) {
-
-  ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
-  uint32_t xfer = 0;
-  std::string fname;
-  ::apache::thrift::protocol::TType ftype;
-  int16_t fid;
-
-  xfer += iprot->readStructBegin(fname);
-
-  using ::apache::thrift::protocol::TProtocolException;
-
-
-  while (true)
-  {
-    xfer += iprot->readFieldBegin(fname, ftype, fid);
-    if (ftype == ::apache::thrift::protocol::T_STOP) {
-      break;
-    }
-    switch (fid)
-    {
-      case 1:
-        if (ftype == ::apache::thrift::protocol::T_STRING) {
-          xfer += iprot->readString(this->start_value);
-          this->__isset.start_value = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      case 2:
-        if (ftype == ::apache::thrift::protocol::T_STRING) {
-          xfer += iprot->readString(this->end_value);
-          this->__isset.end_value = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      default:
-        xfer += iprot->skip(ftype);
-        break;
-    }
-    xfer += iprot->readFieldEnd();
-  }
-
-  xfer += iprot->readStructEnd();
-
-  return xfer;
-}
-
-uint32_t Range::write(::apache::thrift::protocol::TProtocol* oprot) const {
-  uint32_t xfer = 0;
-  ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
-  xfer += oprot->writeStructBegin("Range");
-
-  xfer += oprot->writeFieldBegin("start_value", ::apache::thrift::protocol::T_STRING, 1);
-  xfer += oprot->writeString(this->start_value);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldBegin("end_value", ::apache::thrift::protocol::T_STRING, 2);
-  xfer += oprot->writeString(this->end_value);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldStop();
-  xfer += oprot->writeStructEnd();
-  return xfer;
-}
-
-void swap(Range &a, Range &b) {
-  using ::std::swap;
-  swap(a.start_value, b.start_value);
-  swap(a.end_value, b.end_value);
-  swap(a.__isset, b.__isset);
-}
-
-Range::Range(const Range& other5) {
-  start_value = other5.start_value;
-  end_value = other5.end_value;
-  __isset = other5.__isset;
-}
-Range& Range::operator=(const Range& other6) {
-  start_value = other6.start_value;
-  end_value = other6.end_value;
-  __isset = other6.__isset;
-  return *this;
-}
-void Range::printTo(std::ostream& out) const {
-  using ::apache::thrift::to_string;
-  out << "Range(";
-  out << "start_value=" << to_string(start_value);
-  out << ", " << "end_value=" << to_string(end_value);
-  out << ")";
-}
-
-
-RowRecord::~RowRecord() throw() {
-}
-
-
-void RowRecord::__set_vector_data(const std::string& val) {
-  this->vector_data = val;
-}
-std::ostream& operator<<(std::ostream& out, const RowRecord& obj)
-{
-  obj.printTo(out);
-  return out;
-}
-
-
-uint32_t RowRecord::read(::apache::thrift::protocol::TProtocol* iprot) {
-
-  ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
-  uint32_t xfer = 0;
-  std::string fname;
-  ::apache::thrift::protocol::TType ftype;
-  int16_t fid;
-
-  xfer += iprot->readStructBegin(fname);
-
-  using ::apache::thrift::protocol::TProtocolException;
-
-  bool isset_vector_data = false;
-
-  while (true)
-  {
-    xfer += iprot->readFieldBegin(fname, ftype, fid);
-    if (ftype == ::apache::thrift::protocol::T_STOP) {
-      break;
-    }
-    switch (fid)
-    {
-      case 1:
-        if (ftype == ::apache::thrift::protocol::T_STRING) {
-          xfer += iprot->readBinary(this->vector_data);
-          isset_vector_data = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      default:
-        xfer += iprot->skip(ftype);
-        break;
-    }
-    xfer += iprot->readFieldEnd();
-  }
-
-  xfer += iprot->readStructEnd();
-
-  if (!isset_vector_data)
-    throw TProtocolException(TProtocolException::INVALID_DATA);
-  return xfer;
-}
-
-uint32_t RowRecord::write(::apache::thrift::protocol::TProtocol* oprot) const {
-  uint32_t xfer = 0;
-  ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
-  xfer += oprot->writeStructBegin("RowRecord");
-
-  xfer += oprot->writeFieldBegin("vector_data", ::apache::thrift::protocol::T_STRING, 1);
-  xfer += oprot->writeBinary(this->vector_data);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldStop();
-  xfer += oprot->writeStructEnd();
-  return xfer;
-}
-
-void swap(RowRecord &a, RowRecord &b) {
-  using ::std::swap;
-  swap(a.vector_data, b.vector_data);
-}
-
-RowRecord::RowRecord(const RowRecord& other7) {
-  vector_data = other7.vector_data;
-}
-RowRecord& RowRecord::operator=(const RowRecord& other8) {
-  vector_data = other8.vector_data;
-  return *this;
-}
-void RowRecord::printTo(std::ostream& out) const {
-  using ::apache::thrift::to_string;
-  out << "RowRecord(";
-  out << "vector_data=" << to_string(vector_data);
-  out << ")";
-}
-
-
-QueryResult::~QueryResult() throw() {
-}
-
-
-void QueryResult::__set_id(const int64_t val) {
-  this->id = val;
-}
-
-void QueryResult::__set_distance(const double val) {
-  this->distance = val;
-}
-std::ostream& operator<<(std::ostream& out, const QueryResult& obj)
-{
-  obj.printTo(out);
-  return out;
-}
-
-
-uint32_t QueryResult::read(::apache::thrift::protocol::TProtocol* iprot) {
-
-  ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
-  uint32_t xfer = 0;
-  std::string fname;
-  ::apache::thrift::protocol::TType ftype;
-  int16_t fid;
-
-  xfer += iprot->readStructBegin(fname);
-
-  using ::apache::thrift::protocol::TProtocolException;
-
-
-  while (true)
-  {
-    xfer += iprot->readFieldBegin(fname, ftype, fid);
-    if (ftype == ::apache::thrift::protocol::T_STOP) {
-      break;
-    }
-    switch (fid)
-    {
-      case 1:
-        if (ftype == ::apache::thrift::protocol::T_I64) {
-          xfer += iprot->readI64(this->id);
-          this->__isset.id = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      case 2:
-        if (ftype == ::apache::thrift::protocol::T_DOUBLE) {
-          xfer += iprot->readDouble(this->distance);
-          this->__isset.distance = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      default:
-        xfer += iprot->skip(ftype);
-        break;
-    }
-    xfer += iprot->readFieldEnd();
-  }
-
-  xfer += iprot->readStructEnd();
-
-  return xfer;
-}
-
-uint32_t QueryResult::write(::apache::thrift::protocol::TProtocol* oprot) const {
-  uint32_t xfer = 0;
-  ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
-  xfer += oprot->writeStructBegin("QueryResult");
-
-  xfer += oprot->writeFieldBegin("id", ::apache::thrift::protocol::T_I64, 1);
-  xfer += oprot->writeI64(this->id);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldBegin("distance", ::apache::thrift::protocol::T_DOUBLE, 2);
-  xfer += oprot->writeDouble(this->distance);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldStop();
-  xfer += oprot->writeStructEnd();
-  return xfer;
-}
-
-void swap(QueryResult &a, QueryResult &b) {
-  using ::std::swap;
-  swap(a.id, b.id);
-  swap(a.distance, b.distance);
-  swap(a.__isset, b.__isset);
-}
-
-QueryResult::QueryResult(const QueryResult& other9) {
-  id = other9.id;
-  distance = other9.distance;
-  __isset = other9.__isset;
-}
-QueryResult& QueryResult::operator=(const QueryResult& other10) {
-  id = other10.id;
-  distance = other10.distance;
-  __isset = other10.__isset;
-  return *this;
-}
-void QueryResult::printTo(std::ostream& out) const {
-  using ::apache::thrift::to_string;
-  out << "QueryResult(";
-  out << "id=" << to_string(id);
-  out << ", " << "distance=" << to_string(distance);
-  out << ")";
-}
-
-
-TopKQueryResult::~TopKQueryResult() throw() {
-}
-
-
-void TopKQueryResult::__set_query_result_arrays(const std::vector<QueryResult> & val) {
-  this->query_result_arrays = val;
-}
-std::ostream& operator<<(std::ostream& out, const TopKQueryResult& obj)
-{
-  obj.printTo(out);
-  return out;
-}
-
-
-uint32_t TopKQueryResult::read(::apache::thrift::protocol::TProtocol* iprot) {
-
-  ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
-  uint32_t xfer = 0;
-  std::string fname;
-  ::apache::thrift::protocol::TType ftype;
-  int16_t fid;
-
-  xfer += iprot->readStructBegin(fname);
-
-  using ::apache::thrift::protocol::TProtocolException;
-
-
-  while (true)
-  {
-    xfer += iprot->readFieldBegin(fname, ftype, fid);
-    if (ftype == ::apache::thrift::protocol::T_STOP) {
-      break;
-    }
-    switch (fid)
-    {
-      case 1:
-        if (ftype == ::apache::thrift::protocol::T_LIST) {
-          {
-            this->query_result_arrays.clear();
-            uint32_t _size11;
-            ::apache::thrift::protocol::TType _etype14;
-            xfer += iprot->readListBegin(_etype14, _size11);
-            this->query_result_arrays.resize(_size11);
-            uint32_t _i15;
-            for (_i15 = 0; _i15 < _size11; ++_i15)
-            {
-              xfer += this->query_result_arrays[_i15].read(iprot);
-            }
-            xfer += iprot->readListEnd();
-          }
-          this->__isset.query_result_arrays = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      default:
-        xfer += iprot->skip(ftype);
-        break;
-    }
-    xfer += iprot->readFieldEnd();
-  }
-
-  xfer += iprot->readStructEnd();
-
-  return xfer;
-}
-
-uint32_t TopKQueryResult::write(::apache::thrift::protocol::TProtocol* oprot) const {
-  uint32_t xfer = 0;
-  ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
-  xfer += oprot->writeStructBegin("TopKQueryResult");
-
-  xfer += oprot->writeFieldBegin("query_result_arrays", ::apache::thrift::protocol::T_LIST, 1);
-  {
-    xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->query_result_arrays.size()));
-    std::vector<QueryResult> ::const_iterator _iter16;
-    for (_iter16 = this->query_result_arrays.begin(); _iter16 != this->query_result_arrays.end(); ++_iter16)
-    {
-      xfer += (*_iter16).write(oprot);
-    }
-    xfer += oprot->writeListEnd();
-  }
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldStop();
-  xfer += oprot->writeStructEnd();
-  return xfer;
-}
-
-void swap(TopKQueryResult &a, TopKQueryResult &b) {
-  using ::std::swap;
-  swap(a.query_result_arrays, b.query_result_arrays);
-  swap(a.__isset, b.__isset);
-}
-
-TopKQueryResult::TopKQueryResult(const TopKQueryResult& other17) {
-  query_result_arrays = other17.query_result_arrays;
-  __isset = other17.__isset;
-}
-TopKQueryResult& TopKQueryResult::operator=(const TopKQueryResult& other18) {
-  query_result_arrays = other18.query_result_arrays;
-  __isset = other18.__isset;
-  return *this;
-}
-void TopKQueryResult::printTo(std::ostream& out) const {
-  using ::apache::thrift::to_string;
-  out << "TopKQueryResult(";
-  out << "query_result_arrays=" << to_string(query_result_arrays);
-  out << ")";
-}
-
-
-TopKQueryBinResult::~TopKQueryBinResult() throw() {
-}
-
-
-void TopKQueryBinResult::__set_id_array(const std::string& val) {
-  this->id_array = val;
-}
-
-void TopKQueryBinResult::__set_distance_array(const std::string& val) {
-  this->distance_array = val;
-}
-std::ostream& operator<<(std::ostream& out, const TopKQueryBinResult& obj)
-{
-  obj.printTo(out);
-  return out;
-}
-
-
-uint32_t TopKQueryBinResult::read(::apache::thrift::protocol::TProtocol* iprot) {
-
-  ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
-  uint32_t xfer = 0;
-  std::string fname;
-  ::apache::thrift::protocol::TType ftype;
-  int16_t fid;
-
-  xfer += iprot->readStructBegin(fname);
-
-  using ::apache::thrift::protocol::TProtocolException;
-
-  bool isset_id_array = false;
-  bool isset_distance_array = false;
-
-  while (true)
-  {
-    xfer += iprot->readFieldBegin(fname, ftype, fid);
-    if (ftype == ::apache::thrift::protocol::T_STOP) {
-      break;
-    }
-    switch (fid)
-    {
-      case 1:
-        if (ftype == ::apache::thrift::protocol::T_STRING) {
-          xfer += iprot->readBinary(this->id_array);
-          isset_id_array = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      case 2:
-        if (ftype == ::apache::thrift::protocol::T_STRING) {
-          xfer += iprot->readBinary(this->distance_array);
-          isset_distance_array = true;
-        } else {
-          xfer += iprot->skip(ftype);
-        }
-        break;
-      default:
-        xfer += iprot->skip(ftype);
-        break;
-    }
-    xfer += iprot->readFieldEnd();
-  }
-
-  xfer += iprot->readStructEnd();
-
-  if (!isset_id_array)
-    throw TProtocolException(TProtocolException::INVALID_DATA);
-  if (!isset_distance_array)
-    throw TProtocolException(TProtocolException::INVALID_DATA);
-  return xfer;
-}
-
-uint32_t TopKQueryBinResult::write(::apache::thrift::protocol::TProtocol* oprot) const {
-  uint32_t xfer = 0;
-  ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
-  xfer += oprot->writeStructBegin("TopKQueryBinResult");
-
-  xfer += oprot->writeFieldBegin("id_array", ::apache::thrift::protocol::T_STRING, 1);
-  xfer += oprot->writeBinary(this->id_array);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldBegin("distance_array", ::apache::thrift::protocol::T_STRING, 2);
-  xfer += oprot->writeBinary(this->distance_array);
-  xfer += oprot->writeFieldEnd();
-
-  xfer += oprot->writeFieldStop();
-  xfer += oprot->writeStructEnd();
-  return xfer;
-}
-
-void swap(TopKQueryBinResult &a, TopKQueryBinResult &b) {
-  using ::std::swap;
-  swap(a.id_array, b.id_array);
-  swap(a.distance_array, b.distance_array);
-}
-
-TopKQueryBinResult::TopKQueryBinResult(const TopKQueryBinResult& other19) {
-  id_array = other19.id_array;
-  distance_array = other19.distance_array;
-}
-TopKQueryBinResult& TopKQueryBinResult::operator=(const TopKQueryBinResult& other20) {
-  id_array = other20.id_array;
-  distance_array = other20.distance_array;
-  return *this;
-}
-void TopKQueryBinResult::printTo(std::ostream& out) const {
-  using ::apache::thrift::to_string;
-  out << "TopKQueryBinResult(";
-  out << "id_array=" << to_string(id_array);
-  out << ", " << "distance_array=" << to_string(distance_array);
-  out << ")";
-}
-
-}} // namespace

cpp/src/thrift/gen-cpp/milvus_types.h

@@ -1,394 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.12.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- *  @generated
- */
-#ifndef milvus_TYPES_H
-#define milvus_TYPES_H
-
-#include <iosfwd>
-
-#include <thrift/Thrift.h>
-#include <thrift/TApplicationException.h>
-#include <thrift/TBase.h>
-#include <thrift/protocol/TProtocol.h>
-#include <thrift/transport/TTransport.h>
-
-#include <thrift/stdcxx.h>
-
-
-namespace milvus { namespace thrift {
-
-struct ErrorCode {
-  enum type {
-    SUCCESS = 0,
-    UNEXPECTED_ERROR = 1,
-    CONNECT_FAILED = 2,
-    PERMISSION_DENIED = 3,
-    TABLE_NOT_EXISTS = 4,
-    ILLEGAL_ARGUMENT = 5,
-    ILLEGAL_RANGE = 6,
-    ILLEGAL_DIMENSION = 7,
-    ILLEGAL_INDEX_TYPE = 8,
-    ILLEGAL_TABLE_NAME = 9,
-    ILLEGAL_TOPK = 10,
-    ILLEGAL_ROWRECORD = 11,
-    ILLEGAL_VECTOR_ID = 12,
-    ILLEGAL_SEARCH_RESULT = 13,
-    FILE_NOT_FOUND = 14,
-    META_FAILED = 15,
-    CACHE_FAILED = 16,
-    CANNOT_CREATE_FOLDER = 17,
-    CANNOT_CREATE_FILE = 18,
-    CANNOT_DELETE_FOLDER = 19,
-    CANNOT_DELETE_FILE = 20,
-    BUILD_INDEX_ERROR = 21
-  };
-};
-
-extern const std::map<int, const char*> _ErrorCode_VALUES_TO_NAMES;
-
-std::ostream& operator<<(std::ostream& out, const ErrorCode::type& val);
-
-class Exception;
-
-class TableSchema;
-
-class Range;
-
-class RowRecord;
-
-class QueryResult;
-
-class TopKQueryResult;
-
-class TopKQueryBinResult;
-
-typedef struct _Exception__isset {
-  _Exception__isset() : code(false), reason(false) {}
-  bool code :1;
-  bool reason :1;
-} _Exception__isset;
-
-class Exception : public ::apache::thrift::TException {
- public:
-
-  Exception(const Exception&);
-  Exception& operator=(const Exception&);
-  Exception() : code((ErrorCode::type)0), reason() {
-  }
-
-  virtual ~Exception() throw();
-  ErrorCode::type code;
-  std::string reason;
-
-  _Exception__isset __isset;
-
-  void __set_code(const ErrorCode::type val);
-
-  void __set_reason(const std::string& val);
-
-  bool operator == (const Exception & rhs) const
-  {
-    if (!(code == rhs.code))
-      return false;
-    if (!(reason == rhs.reason))
-      return false;
-    return true;
-  }
-  bool operator != (const Exception &rhs) const {
-    return !(*this == rhs);
-  }
-
-  bool operator < (const Exception & ) const;
-
-  uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
-  uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
-
-  virtual void printTo(std::ostream& out) const;
-  mutable std::string thriftTExceptionMessageHolder_;
-  const char* what() const throw();
-};
-
-void swap(Exception &a, Exception &b);
-
-std::ostream& operator<<(std::ostream& out, const Exception& obj);
-
-typedef struct _TableSchema__isset {
-  _TableSchema__isset() : index_type(true), dimension(true), store_raw_vector(true) {}
-  bool index_type :1;
-  bool dimension :1;
-  bool store_raw_vector :1;
-} _TableSchema__isset;
-
-class TableSchema : public virtual ::apache::thrift::TBase {
- public:
-
-  TableSchema(const TableSchema&);
-  TableSchema& operator=(const TableSchema&);
-  TableSchema() : table_name(), index_type(0), dimension(0LL), store_raw_vector(false) {
-  }
-
-  virtual ~TableSchema() throw();
-  std::string table_name;
-  int32_t index_type;
-  int64_t dimension;
-  bool store_raw_vector;
-
-  _TableSchema__isset __isset;
-
-  void __set_table_name(const std::string& val);
-
-  void __set_index_type(const int32_t val);
-
-  void __set_dimension(const int64_t val);
-
-  void __set_store_raw_vector(const bool val);
-
-  bool operator == (const TableSchema & rhs) const
-  {
-    if (!(table_name == rhs.table_name))
-      return false;
-    if (!(index_type == rhs.index_type))
-      return false;
-    if (!(dimension == rhs.dimension))
-      return false;
-    if (!(store_raw_vector == rhs.store_raw_vector))
-      return false;
-    return true;
-  }
-  bool operator != (const TableSchema &rhs) const {
-    return !(*this == rhs);
-  }
-
-  bool operator < (const TableSchema & ) const;
-
-  uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
-  uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
-
-  virtual void printTo(std::ostream& out) const;
-};
-
-void swap(TableSchema &a, TableSchema &b);
-
-std::ostream& operator<<(std::ostream& out, const TableSchema& obj);
-
-typedef struct _Range__isset {
-  _Range__isset() : start_value(false), end_value(false) {}
-  bool start_value :1;
-  bool end_value :1;
-} _Range__isset;
-
-class Range : public virtual ::apache::thrift::TBase {
- public:
-
-  Range(const Range&);
-  Range& operator=(const Range&);
-  Range() : start_value(), end_value() {
-  }
-
-  virtual ~Range() throw();
-  std::string start_value;
-  std::string end_value;
-
-  _Range__isset __isset;
-
-  void __set_start_value(const std::string& val);
-
-  void __set_end_value(const std::string& val);
-
-  bool operator == (const Range & rhs) const
-  {
-    if (!(start_value == rhs.start_value))
-      return false;
-    if (!(end_value == rhs.end_value))
-      return false;
-    return true;
-  }
-  bool operator != (const Range &rhs) const {
-    return !(*this == rhs);
-  }
-
-  bool operator < (const Range & ) const;
-
-  uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
-  uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
-
-  virtual void printTo(std::ostream& out) const;
-};
-
-void swap(Range &a, Range &b);
-
-std::ostream& operator<<(std::ostream& out, const Range& obj);
-
-
-class RowRecord : public virtual ::apache::thrift::TBase {
- public:
-
-  RowRecord(const RowRecord&);
-  RowRecord& operator=(const RowRecord&);
-  RowRecord() : vector_data() {
-  }
-
-  virtual ~RowRecord() throw();
-  std::string vector_data;
-
-  void __set_vector_data(const std::string& val);
-
-  bool operator == (const RowRecord & rhs) const
-  {
-    if (!(vector_data == rhs.vector_data))
-      return false;
-    return true;
-  }
-  bool operator != (const RowRecord &rhs) const {
-    return !(*this == rhs);
-  }
-
-  bool operator < (const RowRecord & ) const;
-
-  uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
-  uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
-
-  virtual void printTo(std::ostream& out) const;
-};
-
-void swap(RowRecord &a, RowRecord &b);
-
-std::ostream& operator<<(std::ostream& out, const RowRecord& obj);
-
-typedef struct _QueryResult__isset {
-  _QueryResult__isset() : id(false), distance(false) {}
-  bool id :1;
-  bool distance :1;
-} _QueryResult__isset;
-
-class QueryResult : public virtual ::apache::thrift::TBase {
- public:
-
-  QueryResult(const QueryResult&);
-  QueryResult& operator=(const QueryResult&);
-  QueryResult() : id(0), distance(0) {
-  }
-
-  virtual ~QueryResult() throw();
-  int64_t id;
-  double distance;
-
-  _QueryResult__isset __isset;
-
-  void __set_id(const int64_t val);
-
-  void __set_distance(const double val);
-
-  bool operator == (const QueryResult & rhs) const
-  {
-    if (!(id == rhs.id))
-      return false;
-    if (!(distance == rhs.distance))
-      return false;
-    return true;
-  }
-  bool operator != (const QueryResult &rhs) const {
-    return !(*this == rhs);
-  }
-
-  bool operator < (const QueryResult & ) const;
-
-  uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
-  uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
-
-  virtual void printTo(std::ostream& out) const;
-};
-
-void swap(QueryResult &a, QueryResult &b);
-
-std::ostream& operator<<(std::ostream& out, const QueryResult& obj);
-
-typedef struct _TopKQueryResult__isset {
-  _TopKQueryResult__isset() : query_result_arrays(false) {}
-  bool query_result_arrays :1;
-} _TopKQueryResult__isset;
-
-class TopKQueryResult : public virtual ::apache::thrift::TBase {
- public:
-
-  TopKQueryResult(const TopKQueryResult&);
-  TopKQueryResult& operator=(const TopKQueryResult&);
-  TopKQueryResult() {
-  }
-
-  virtual ~TopKQueryResult() throw();
-  std::vector<QueryResult>  query_result_arrays;
-
-  _TopKQueryResult__isset __isset;
-
-  void __set_query_result_arrays(const std::vector<QueryResult> & val);
-
-  bool operator == (const TopKQueryResult & rhs) const
-  {
-    if (!(query_result_arrays == rhs.query_result_arrays))
-      return false;
-    return true;
-  }
-  bool operator != (const TopKQueryResult &rhs) const {
-    return !(*this == rhs);
-  }
-
-  bool operator < (const TopKQueryResult & ) const;
-
-  uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
-  uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
-
-  virtual void printTo(std::ostream& out) const;
-};
-
-void swap(TopKQueryResult &a, TopKQueryResult &b);
-
-std::ostream& operator<<(std::ostream& out, const TopKQueryResult& obj);
-
-
-class TopKQueryBinResult : public virtual ::apache::thrift::TBase {
- public:
-
-  TopKQueryBinResult(const TopKQueryBinResult&);
-  TopKQueryBinResult& operator=(const TopKQueryBinResult&);
-  TopKQueryBinResult() : id_array(), distance_array() {
-  }
-
-  virtual ~TopKQueryBinResult() throw();
-  std::string id_array;
-  std::string distance_array;
-
-  void __set_id_array(const std::string& val);
-
-  void __set_distance_array(const std::string& val);
-
-  bool operator == (const TopKQueryBinResult & rhs) const
-  {
-    if (!(id_array == rhs.id_array))
-      return false;
-    if (!(distance_array == rhs.distance_array))
-      return false;
-    return true;
-  }
-  bool operator != (const TopKQueryBinResult &rhs) const {
-    return !(*this == rhs);
-  }
-
-  bool operator < (const TopKQueryBinResult & ) const;
-
-  uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
-  uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
-
-  virtual void printTo(std::ostream& out) const;
-};
-
-void swap(TopKQueryBinResult &a, TopKQueryBinResult &b);
-
-std::ostream& operator<<(std::ostream& out, const TopKQueryBinResult& obj);
-
-}} // namespace
-
-#endif

cpp/src/thrift/milvus.thrift

@@ -1,246 +0,0 @@
-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-namespace cpp milvus.thrift
-namespace py milvus.thrift
-namespace d milvus.thrift
-namespace dart milvus.thrift
-namespace java milvus.thrift
-namespace perl milvus.thrift
-namespace php milvus.thrift
-namespace haxe milvus.thrift
-namespace netcore milvus.thrift
-
-enum ErrorCode {
-    SUCCESS = 0,
-    UNEXPECTED_ERROR,
-    CONNECT_FAILED,
-    PERMISSION_DENIED,
-    TABLE_NOT_EXISTS,
-    ILLEGAL_ARGUMENT,
-    ILLEGAL_RANGE,
-    ILLEGAL_DIMENSION,
-    ILLEGAL_INDEX_TYPE,
-    ILLEGAL_TABLE_NAME,
-    ILLEGAL_TOPK,
-    ILLEGAL_ROWRECORD,
-    ILLEGAL_VECTOR_ID,
-    ILLEGAL_SEARCH_RESULT,
-    FILE_NOT_FOUND,
-    META_FAILED,
-    CACHE_FAILED,
-    CANNOT_CREATE_FOLDER,
-    CANNOT_CREATE_FILE,
-    CANNOT_DELETE_FOLDER,
-    CANNOT_DELETE_FILE,
-    BUILD_INDEX_ERROR,
-}
-
-exception Exception {
-    1: ErrorCode code;
-    2: string reason;
-}
-
-
-/**
- * @brief Table Schema
- */
-struct TableSchema {
-    1: required string table_name;                   ///< Table name
-    2: i32 index_type = 0;                           ///< Index type, optional: 0-invalid, 1-idmap, 2-ivflat
-    3: i64 dimension = 0;                            ///< Vector dimension
-    4: bool store_raw_vector = false;                ///< Store raw data
-}
-
-/**
- * @brief Range Schema
- */
-struct Range {
-    1: string start_value;                           ///< Range start
-    2: string end_value;                             ///< Range stop
-}
-
-/**
- * @brief Record inserted
- */
-struct RowRecord {
-    1: required binary vector_data;                  ///< Vector data, double array
-}
-
-/**
- * @brief Query result
- */
-struct QueryResult {
-    1: i64 id;                                       ///< Output result
-    2: double distance;                              ///< Vector similarity distance
-}
-
-/**
- * @brief TopK query result
- */
-struct TopKQueryResult {
-    1: list<QueryResult> query_result_arrays;        ///< TopK query result
-}
-
-/**
- * @brief TopK query binary result
- */
-struct TopKQueryBinResult {
-    1: required binary id_array;                     ///< id array, interger array
-    2: required binary distance_array;               ///< distance array, double array
-}
-
-service MilvusService {
-    /**
-     * @brief Create table method
-     *
-     * This method is used to create table
-     *
-     * @param param, use to provide table information to be created.
-     *
-     */
-    void CreateTable(2: TableSchema param) throws(1: Exception e);
-
-    /**
-     * @brief Test table existence method
-     *
-     * This method is used to test table existence.
-     *
-     * @param table_name, table name is going to be tested.
-     *
-     */
-    bool HasTable(2: string table_name) throws(1: Exception e);
-
-
-    /**
-     * @brief Delete table method
-     *
-     * This method is used to delete table.
-     *
-     * @param table_name, table name is going to be deleted.
-     *
-     */
-    void DeleteTable(2: string table_name) throws(1: Exception e);
-
-    /**
-     * @brief Build index by table method
-     *
-     * This method is used to build index by table in sync mode.
-     *
-     * @param table_name, table is going to be built index.
-     *
-     */
-    void BuildIndex(2: string table_name) throws(1: Exception e);
-
-
-    /**
-     * @brief Add vector array to table
-     *
-     * This method is used to add vector array to table.
-     *
-     * @param table_name, table_name is inserted.
-     * @param record_array, vector array is inserted.
-     *
-     * @return vector id array
-     */
-    list<i64> AddVector(2: string table_name,
-                        3: list<RowRecord> record_array) throws(1: Exception e);
-
-
-    /**
-     * @brief Query vector
-     *
-     * This method is used to query vector in table.
-     *
-     * @param table_name, table_name is queried.
-     * @param query_record_array, all vector are going to be queried.
-     * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-     * @param topk, how many similarity vectors will be searched.
-     *
-     * @return query result array.
-     */
-    list<TopKQueryResult> SearchVector(2: string table_name,
-                                       3: list<RowRecord> query_record_array,
-                                       4: list<Range> query_range_array,
-                                       5: i64 topk) throws(1: Exception e);
-
-    /**
-     * @brief Query vector
-     *
-     * This method is used to query vector in table.
-     *
-     * @param table_name, table_name is queried.
-     * @param query_record_array, all vector are going to be queried.
-     * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-     * @param topk, how many similarity vectors will be searched.
-     *
-     * @return query binary result array.
-     */
-    list<TopKQueryBinResult> SearchVector2(2: string table_name,
-                                           3: list<RowRecord> query_record_array,
-                                           4: list<Range> query_range_array,
-                                           5: i64 topk) throws(1: Exception e);
-
-    /**
-     * @brief Internal use query interface
-     *
-     * This method is used to query vector in specified files.
-     *
-     * @param file_id_array, specified files id array, queried.
-     * @param query_record_array, all vector are going to be queried.
-     * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-     * @param topk, how many similarity vectors will be searched.
-     *
-     * @return query result array.
-     */
-    list<TopKQueryResult> SearchVectorInFiles(2: string table_name,
-                                              3: list<string> file_id_array,
-                                              4: list<RowRecord> query_record_array,
-                                              5: list<Range> query_range_array,
-                                              6: i64 topk) throws(1: Exception e);
-
-    /**
-     * @brief Get table schema
-     *
-     * This method is used to get table schema.
-     *
-     * @param table_name, target table name.
-     *
-     * @return table schema
-     */
-    TableSchema DescribeTable(2: string table_name) throws(1: Exception e);
-
-
-    /**
-     * @brief Get table row count
-     *
-     * This method is used to get table row count.
-     *
-     * @param table_name, target table name.
-     *
-     * @return table row count
-     */
-    i64 GetTableRowCount(2: string table_name) throws(1: Exception e);
-
-    /**
-     * @brief List all tables in database
-     *
-     * This method is used to list all tables.
-     *
-     *
-     * @return table names.
-     */
-    list<string> ShowTables() throws(1: Exception e);
-
-
-    /**
-     * @brief Give the server status
-     *
-     * This method is used to give the server status.
-     *
-     * @return Server status.
-     */
-    string Ping(2: string cmd) throws(1: Exception e);
-}

cpp/src/utils/ValidationUtil.cpp

@@ -76,7 +76,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
 }
 
 ServerError
-ValidationUtil::ValidateTableIndexFileSize(int32_t index_file_size) {
+ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
     if(index_file_size <= 0 || index_file_size > index_file_size_limit) {
         return SERVER_INVALID_INDEX_FILE_SIZE;
     }

cpp/src/utils/ValidationUtil.h

@@ -21,7 +21,7 @@ public:
     ValidateTableIndexNlist(int32_t nlist);
 
     static ServerError
-    ValidateTableIndexFileSize(int32_t index_file_size);
+    ValidateTableIndexFileSize(int64_t index_file_size);
 
     static ServerError
     ValidateTableIndexMetricType(int32_t metric_type);

cpp/thirdparty/versions.txt

@@ -15,7 +15,6 @@ ROCKSDB_VERSION=v6.0.2
 SNAPPY_VERSION=1.1.7
 SQLITE_VERSION=3280000
 SQLITE_ORM_VERSION=master
-THRIFT_VERSION=v0.12.0
 YAMLCPP_VERSION=0.6.2
 ZLIB_VERSION=v1.2.11
 ZSTD_VERSION=v1.4.0

cpp/unittest/db/CMakeLists.txt

@@ -10,6 +10,12 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
 aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src)
+aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
+
 aux_source_directory(./ test_srcs)
 
 set(util_files
@@ -40,6 +46,11 @@ set(db_test_src
     ${db_meta_files}
     ${db_scheduler_srcs}
     ${wrapper_src}
+    ${scheduler_action_srcs}
+    ${scheduler_event_srcs}
+    ${scheduler_resource_srcs}
+    ${scheduler_task_srcs}
+    ${scheduler_srcs}
     ${knowhere_src}
     ${util_files}
     ${require_files}

cpp/unittest/db/scheduler_test.cpp

@@ -100,7 +100,7 @@ TEST(DBSchedulerTest, DELETE_SCHEDULER_TEST) {
     }
 
     engine::meta::Meta::Ptr meta_ptr;
-    engine::DeleteContextPtr context_ptr = std::make_shared<engine::DeleteContext>(table_id, meta_ptr);
+    engine::DeleteContextPtr context_ptr = std::make_shared<engine::DeleteContext>(table_id, meta_ptr, 0);
     ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list);
     ASSERT_TRUE(ret);
     ASSERT_EQ(task_list.size(), 21);
@@ -115,7 +115,7 @@ TEST(DBSchedulerTest, DELETE_SCHEDULER_TEST) {
         }
     }
 
-    context_ptr = std::make_shared<engine::DeleteContext>("no_task_table", meta_ptr);
+    context_ptr = std::make_shared<engine::DeleteContext>("no_task_table", meta_ptr, 0);
     ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list);
     ASSERT_TRUE(ret);
     ASSERT_EQ(task_list.size(), 22);

cpp/unittest/db/utils.cpp

@@ -59,12 +59,27 @@ engine::Options DBTest::GetOptions() {
 
 void DBTest::SetUp() {
     InitLog();
+
+    auto res_mgr = engine::ResMgrInst::GetInstance();
+    res_mgr->Clear();
+    res_mgr->Add(engine::ResourceFactory::Create("disk", "DISK", 0, true, false));
+    res_mgr->Add(engine::ResourceFactory::Create("cpu", "CPU", 0, true, true));
+
+    auto default_conn = engine::Connection("IO", 500.0);
+    res_mgr->Connect("disk", "cpu", default_conn);
+    res_mgr->Start();
+    engine::SchedInst::GetInstance()->Start();
+
     auto options = GetOptions();
     db_ = engine::DBFactory::Build(options);
 }
 
 void DBTest::TearDown() {
     delete db_;
+
+    engine::ResMgrInst::GetInstance()->Stop();
+    engine::SchedInst::GetInstance()->Stop();
+
     boost::filesystem::remove_all("/tmp/milvus_test");
 }
 
@@ -117,6 +132,21 @@ void NewMemManagerTest::InitLog() {
 
 void NewMemManagerTest::SetUp() {
     InitLog();
+
+    auto res_mgr = engine::ResMgrInst::GetInstance();
+    res_mgr->Clear();
+    res_mgr->Add(engine::ResourceFactory::Create("disk", "DISK", 0, true, false));
+    res_mgr->Add(engine::ResourceFactory::Create("cpu", "CPU", 0, true, true));
+
+    auto default_conn = engine::Connection("IO", 500.0);
+    res_mgr->Connect("disk", "cpu", default_conn);
+    res_mgr->Start();
+    engine::SchedInst::GetInstance()->Start();
+}
+
+void NewMemManagerTest::TearDown() {
+    engine::ResMgrInst::GetInstance()->Stop();
+    engine::SchedInst::GetInstance()->Stop();
 }
 
 int main(int argc, char **argv) {

cpp/unittest/db/utils.h

@@ -13,6 +13,8 @@
 #include "db/DB.h"
 #include "db/meta/SqliteMetaImpl.h"
 #include "db/meta/MySQLMetaImpl.h"
+#include "scheduler/SchedInst.h"
+#include "scheduler/ResourceFactory.h"
 
 
 #define TIMING
@@ -91,4 +93,5 @@ class DISABLED_MySQLDBTest : public ::testing::Test {
 class NewMemManagerTest : public ::testing::Test {
     void InitLog();
     void SetUp() override;
+    void TearDown() override;
 };

cpp/unittest/metrics/CMakeLists.txt

@@ -19,6 +19,11 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
 aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src)
 aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src)
+aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs)
+aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
 aux_source_directory(${MILVUS_ENGINE_SRC}/src/metrics metrics_src)
 aux_source_directory(./ test_srcs)
 
@@ -52,6 +57,11 @@ set(count_test_src
         ${db_meta_files}
         ${db_scheduler_srcs}
         ${wrapper_src}
+        ${scheduler_action_srcs}
+        ${scheduler_event_srcs}
+        ${scheduler_resource_srcs}
+        ${scheduler_task_srcs}
+        ${scheduler_srcs}
         ${knowhere_src}
         ${metrics_src}
         ${test_srcs}

cpp/unittest/server/config_test.cpp

@@ -135,12 +135,6 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) {
     err = config.ValidateConfig();
     ASSERT_NE(err, server::SERVER_SUCCESS);
 
-    size_t index_building_threshold = (gpu_mem + 1*MB)/MB;
-    db_config.SetValue(server::CONFIG_DB_INDEX_TRIGGER_SIZE,
-            std::to_string(index_building_threshold));
-    err = config.ValidateConfig();
-    ASSERT_NE(err, server::SERVER_SUCCESS);
-
     insert_buffer_size = total_mem/GB + 2;
     db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size));
     err = config.ValidateConfig();