diff --git a/CHANGELOGS.md b/CHANGELOGS.md index a5d7bfec58..d9ef3d4c36 100644 --- a/CHANGELOGS.md +++ b/CHANGELOGS.md @@ -15,3 +15,5 @@ Please mark all change in change log and use the ticket from JIRA. ### Task - MS-1 - Add CHANGELOG.md +- MS-161 - Add CI / CD Module to Milvus Project +- MS-202 - Add Milvus Jenkins project email notification diff --git a/ci/function/file_transfer.groovy b/ci/function/file_transfer.groovy new file mode 100644 index 0000000000..bebae14832 --- /dev/null +++ b/ci/function/file_transfer.groovy @@ -0,0 +1,10 @@ +def FileTransfer (sourceFiles, remoteDirectory, remoteIP, protocol = "ftp", makeEmptyDirs = true) { + if (protocol == "ftp") { + ftpPublisher masterNodeName: '', paramPublish: [parameterName: ''], alwaysPublishFromMaster: false, continueOnError: false, failOnError: true, publishers: [ + [configName: "${remoteIP}", transfers: [ + [asciiMode: false, cleanRemote: false, excludes: '', flatten: false, makeEmptyDirs: "${makeEmptyDirs}", noDefaultExcludes: false, patternSeparator: '[, ]+', remoteDirectory: "${remoteDirectory}", remoteDirectorySDF: false, removePrefix: '', sourceFiles: "${sourceFiles}"]], usePromotionTimestamp: true, useWorkspaceInPromotion: false, verbose: true + ] + ] + } +} +return this diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy new file mode 100644 index 0000000000..2e9332fa6e --- /dev/null +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -0,0 +1,13 @@ +try { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } +} catch (exc) { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } + throw exc +} + diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy new file mode 100644 index 0000000000..e57988fefe --- /dev/null +++ b/ci/jenkinsfile/cluster_cleanup_dev.groovy @@ -0,0 +1,13 @@ +try { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" + } +} catch (exc) { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" + } + throw exc +} + diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy new file mode 100644 index 0000000000..cde361b9ea --- /dev/null +++ b/ci/jenkinsfile/cluster_deploy2dev.groovy @@ -0,0 +1,22 @@ +try { + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' + sh 'helm repo update' + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("milvus/milvus-cluster") { + sh "helm install --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.3.1 . " + } + } + timeout(time: 2, unit: 'MINUTES') { + waitUntil { + def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true + return !result + } + } +} catch (exc) { + echo 'Helm running failed!' + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" + throw exc +} + diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy new file mode 100644 index 0000000000..8093f67d18 --- /dev/null +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -0,0 +1,12 @@ +timeout(time: 10, unit: 'MINUTES') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements.txt' + sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" + } + } catch (exc) { + echo 'Milvus Test Failed !' + throw exc + } +} diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy new file mode 100644 index 0000000000..b0c537f2e1 --- /dev/null +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -0,0 +1,22 @@ +try { + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' + sh 'helm repo update' + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("milvus/milvus-gpu") { + sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.1 ." + } + } + timeout(time: 2, unit: 'MINUTES') { + waitUntil { + def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true + return !result + } + } +} catch (exc) { + echo 'Helm running failed!' + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + throw exc +} + diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy new file mode 100644 index 0000000000..836ddbad05 --- /dev/null +++ b/ci/jenkinsfile/dev_test.groovy @@ -0,0 +1,12 @@ +timeout(time: 10, unit: 'MINUTES') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements.txt' + sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" + } + } catch (exc) { + echo 'Milvus Test Failed !' + throw exc + } +} diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy new file mode 100644 index 0000000000..99892c90ef --- /dev/null +++ b/ci/jenkinsfile/milvus_build.groovy @@ -0,0 +1,20 @@ +container('milvus-build-env') { + timeout(time: 20, unit: 'MINUTES') { + gitlabCommitStatus(name: 'Build Engine') { + dir ("milvus_engine") { + try { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("cpp") { + sh "git config --global user.email \"test@zilliz.com\"" + sh "git config --global user.name \"test\"" + sh "./build.sh -t ${params.BUILD_TYPE} -u -c" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Build Engine', state: 'failed' + throw exc + } + } + } + } +} + diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy new file mode 100644 index 0000000000..6c30ce491c --- /dev/null +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -0,0 +1,20 @@ +container('milvus-build-env') { + timeout(time: 20, unit: 'MINUTES') { + gitlabCommitStatus(name: 'Build Engine') { + dir ("milvus_engine") { + try { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("cpp") { + sh "git config --global user.email \"test@zilliz.com\"" + sh "git config --global user.name \"test\"" + sh "./build.sh -t ${params.BUILD_TYPE}" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Build Engine', state: 'failed' + throw exc + } + } + } + } +} + diff --git a/ci/jenkinsfile/notify.groovy b/ci/jenkinsfile/notify.groovy new file mode 100644 index 0000000000..0a257b8cd8 --- /dev/null +++ b/ci/jenkinsfile/notify.groovy @@ -0,0 +1,15 @@ +def notify() { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } +} +return this + diff --git a/ci/jenkinsfile/packaged_milvus.groovy b/ci/jenkinsfile/packaged_milvus.groovy new file mode 100644 index 0000000000..407b100589 --- /dev/null +++ b/ci/jenkinsfile/packaged_milvus.groovy @@ -0,0 +1,44 @@ +container('milvus-build-env') { + timeout(time: 5, unit: 'MINUTES') { + dir ("milvus_engine") { + dir ("cpp") { + gitlabCommitStatus(name: 'Packaged Engine') { + if (fileExists('milvus')) { + try { + sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\"" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' + throw exc + } + } else { + updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' + error("Milvus binary directory don't exists!") + } + } + + gitlabCommitStatus(name: 'Packaged Engine lcov') { + if (fileExists('lcov_out')) { + try { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("lcov_out/", "${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus lcov out Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}/lcov_out/\"" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed' + throw exc + } + } else { + updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed' + error("Milvus lcov out directory don't exists!") + } + } + } + } + } +} diff --git a/ci/jenkinsfile/packaged_milvus_no_ut.groovy b/ci/jenkinsfile/packaged_milvus_no_ut.groovy new file mode 100644 index 0000000000..b6c31540a1 --- /dev/null +++ b/ci/jenkinsfile/packaged_milvus_no_ut.groovy @@ -0,0 +1,26 @@ +container('milvus-build-env') { + timeout(time: 5, unit: 'MINUTES') { + dir ("milvus_engine") { + dir ("cpp") { + gitlabCommitStatus(name: 'Packaged Engine') { + if (fileExists('milvus')) { + try { + sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\"" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' + throw exc + } + } else { + updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' + error("Milvus binary directory don't exists!") + } + } + } + } + } +} diff --git a/ci/jenkinsfile/publish_docker.groovy b/ci/jenkinsfile/publish_docker.groovy new file mode 100644 index 0000000000..7416c9cb17 --- /dev/null +++ b/ci/jenkinsfile/publish_docker.groovy @@ -0,0 +1,32 @@ +container('publish-docker') { + timeout(time: 15, unit: 'MINUTES') { + gitlabCommitStatus(name: 'Publish Engine Docker') { + try { + dir ("${PROJECT_NAME}_build") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("docker/deploy/ubuntu16.04/free_version") { + sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" + sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" + try { + docker.withRegistry('https://registry.zilliz.com', 'a54e38ef-c424-4ea9-9224-b25fc20e3924') { + def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}") + customImage.push() + } + echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}" + } catch (exc) { + updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled' + throw exc + } finally { + sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}" + } + } + } + } catch (exc) { + updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed' + echo 'Publish docker failed!' + throw exc + } + } + } +} + diff --git a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy new file mode 100644 index 0000000000..6bbd8a649f --- /dev/null +++ b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy @@ -0,0 +1,14 @@ +timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + if (fileExists('cluster_test_out')) { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("cluster_test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" + } + } else { + error("Milvus Dev Test Out directory don't exists!") + } + } +} + diff --git a/ci/jenkinsfile/upload_dev_test_out.groovy b/ci/jenkinsfile/upload_dev_test_out.groovy new file mode 100644 index 0000000000..7e106c0296 --- /dev/null +++ b/ci/jenkinsfile/upload_dev_test_out.groovy @@ -0,0 +1,13 @@ +timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + if (fileExists('test_out')) { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" + } + } else { + error("Milvus Dev Test Out directory don't exists!") + } + } +} diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile new file mode 100644 index 0000000000..9577837bfd --- /dev/null +++ b/ci/main_jenkinsfile @@ -0,0 +1,365 @@ +pipeline { + agent none + + options { + timestamps() + } + + environment { + PROJECT_NAME = "milvus" + LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() + SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" + GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" + SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" + DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" + } + + stages { + stage("Ubuntu 16.04") { + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' + ]); + + DOCKER_VERSION = VersionNumber([ + versionNumberString : '${DOCKER_VERSION_STR}' + ]); + } + + stages { + stage("Run Build") { + agent { + kubernetes { + cloud 'build-kubernetes' + label 'build' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-build-env' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Build') { + steps { + gitlabCommitStatus(name: 'Build') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Build', state: 'canceled' + echo "Milvus Build aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Build', state: 'failed' + echo "Milvus Build failure !" + } + } + } + } + + stage("Publish docker and helm") { + agent { + kubernetes { + label 'publish' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-docker + image: registry.zilliz.com/library/zilliz_docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock +""" + } + } + stages { + stage('Publish Docker') { + steps { + gitlabCommitStatus(name: 'Publish Docker') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' + echo "Milvus Publish Docker aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' + echo "Milvus Publish Docker failure !" + } + } + } + } + + stage("Deploy to Development") { + parallel { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + + stages { + stage("Deploy to Dev") { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + } + } + } + } + } + stage("Dev Test") { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + } + } + } + } + } + stage ("Cleanup Dev") { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + success { + script { + echo "Milvus Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Single Node CI/CD failure !" + } + } + } + } + + stage("Cluster") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + stages { + stage("Deploy to Dev") { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" + } + } + } + } + } + stage("Dev Test") { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" + } + } + } + } + } + stage ("Cleanup Dev") { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + success { + script { + echo "Milvus Cluster CI/CD success !" + } + } + aborted { + script { + echo "Milvus Cluster CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Cluster CI/CD failure !" + } + } + } + } + } + } + } + } + } + + post { + always { + script { + if (env.gitlabAfter != null) { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } + } + } + } + + success { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'success' + echo "Milvus CI/CD success !" + } + } + + aborted { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' + echo "Milvus CI/CD aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'failed' + echo "Milvus CI/CD failure !" + } + } + } +} + diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut new file mode 100644 index 0000000000..082433650c --- /dev/null +++ b/ci/main_jenkinsfile_no_ut @@ -0,0 +1,365 @@ +pipeline { + agent none + + options { + timestamps() + } + + environment { + PROJECT_NAME = "milvus" + LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() + SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" + GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" + SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" + DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" + } + + stages { + stage("Ubuntu 16.04") { + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' + ]); + + DOCKER_VERSION = VersionNumber([ + versionNumberString : '${DOCKER_VERSION_STR}' + ]); + } + + stages { + stage("Run Build") { + agent { + kubernetes { + cloud 'build-kubernetes' + label 'build' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-build-env' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Build') { + steps { + gitlabCommitStatus(name: 'Build') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build_no_ut.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus_no_ut.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Build', state: 'canceled' + echo "Milvus Build aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Build', state: 'failed' + echo "Milvus Build failure !" + } + } + } + } + + stage("Publish docker and helm") { + agent { + kubernetes { + label 'publish' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-docker + image: registry.zilliz.com/library/zilliz_docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock +""" + } + } + stages { + stage('Publish Docker') { + steps { + gitlabCommitStatus(name: 'Publish Docker') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' + echo "Milvus Publish Docker aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' + echo "Milvus Publish Docker failure !" + } + } + } + } + + stage("Deploy to Development") { + parallel { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + + stages { + stage("Deploy to Dev") { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + } + } + } + } + } + stage("Dev Test") { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + } + } + } + } + } + stage ("Cleanup Dev") { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + success { + script { + echo "Milvus Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Single Node CI/CD failure !" + } + } + } + } + + stage("Cluster") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + stages { + stage("Deploy to Dev") { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" + } + } + } + } + } + stage("Dev Test") { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" + } + } + } + } + } + stage ("Cleanup Dev") { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + success { + script { + echo "Milvus Cluster CI/CD success !" + } + } + aborted { + script { + echo "Milvus Cluster CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Cluster CI/CD failure !" + } + } + } + } + } + } + } + } + } + + post { + always { + script { + if (env.gitlabAfter != null) { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } + } + } + } + + success { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'success' + echo "Milvus CI/CD success !" + } + } + + aborted { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' + echo "Milvus CI/CD aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'failed' + echo "Milvus CI/CD failure !" + } + } + } +} + diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile new file mode 100644 index 0000000000..7f59561f82 --- /dev/null +++ b/ci/nightly_main_jenkinsfile @@ -0,0 +1,363 @@ +pipeline { + agent none + + options { + timestamps() + } + + environment { + PROJECT_NAME = "milvus" + LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() + SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" + GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" + SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" + DOCKER_VERSION_STR = "${env.gitlabAfter == null ? '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, \"yyyyMMdd\"}' : '${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}'}" + } + + stages { + stage("Ubuntu 16.04") { + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' + ]); + + DOCKER_VERSION = VersionNumber([ + versionNumberString : '${DOCKER_VERSION_STR}' + ]); + } + + stages { + stage("Run Build") { + agent { + kubernetes { + cloud 'build-kubernetes' + label 'build' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-build-env' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Build') { + steps { + gitlabCommitStatus(name: 'Build') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Build', state: 'canceled' + echo "Milvus Build aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Build', state: 'failed' + echo "Milvus Build failure !" + } + } + } + } + + stage("Publish docker and helm") { + agent { + kubernetes { + label 'publish' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-docker + image: registry.zilliz.com/library/zilliz_docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock +""" + } + } + stages { + stage('Publish Docker') { + steps { + gitlabCommitStatus(name: 'Publish Docker') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' + echo "Milvus Publish Docker aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' + echo "Milvus Publish Docker failure !" + } + } + } + } + + stage("Deploy to Development") { + parallel { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + + stages { + stage("Deploy to Dev") { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + } + } + } + } + } + stage("Dev Test") { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + } + } + } + } + } + stage ("Cleanup Dev") { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + success { + script { + echo "Milvus Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Single Node CI/CD failure !" + } + } + } + } + + stage("Cluster") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + stages { + stage("Deploy to Dev") { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" + } + } + } + } + } + stage("Dev Test") { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" + } + } + } + } + } + stage ("Cleanup Dev") { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + success { + script { + echo "Milvus Cluster CI/CD success !" + } + } + aborted { + script { + echo "Milvus Cluster CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Cluster CI/CD failure !" + } + } + } + } + } + } + } + } + } + + post { + always { + script { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } + } + } + + success { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'success' + echo "Milvus CI/CD success !" + } + } + + aborted { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' + echo "Milvus CI/CD aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'failed' + echo "Milvus CI/CD failure !" + } + } + } +} + diff --git a/ci/pod_containers/milvus-engine-build.yaml b/ci/pod_containers/milvus-engine-build.yaml new file mode 100644 index 0000000000..cd5352ffef --- /dev/null +++ b/ci/pod_containers/milvus-engine-build.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: build-env +spec: + containers: + - name: milvus-build-env + image: registry.zilliz.com/milvus/milvus-build-env:v0.9 + command: + - cat + tty: true diff --git a/ci/pod_containers/milvus-testframework.yaml b/ci/pod_containers/milvus-testframework.yaml new file mode 100644 index 0000000000..7a98fbca8e --- /dev/null +++ b/ci/pod_containers/milvus-testframework.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: testframework +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.1 + command: + - cat + tty: true diff --git a/ci/pod_containers/publish-docker.yaml b/ci/pod_containers/publish-docker.yaml new file mode 100644 index 0000000000..268afb1331 --- /dev/null +++ b/ci/pod_containers/publish-docker.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-docker + image: registry.zilliz.com/library/zilliz_docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 0f4e480123..7e7f88a2e9 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -11,13 +11,21 @@ Please mark all change in change log and use the ticket from JIRA. - MS-149 - Fixed searching only one index file issue in distributed mode - MS-153 - fix c_str error when connecting to MySQL - MS-157 - fix changelog +- MS-190 - use env variable to switch mem manager and fix cmake ## Improvement - MS-156 - Add unittest for merge result functions - - MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl +- MS-204 - Support multi db_path +- MS-206 - Support SQ8 index type +- MS-208 - Add buildinde interface for C++ SDK +- MS-212 - Support Inner product metric type ## New Feature +- MS-195 - Add nlist and use_blas_threshold conf + +## New Feature +- MS-180 - Add new mem manager ## Task @@ -72,6 +80,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-144 - Add nprobe config - MS-147 - Enable IVF +- MS-130 - Add prometheus_test ## Task - MS-74 - Change README.md in cpp - MS-88 - Add support for arm architecture diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 9aa3f62124..9e19b037f8 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -525,7 +525,7 @@ if(MILVUS_BOOST_VENDORED) "" ${EP_LOG_OPTIONS}) set(Boost_INCLUDE_DIR "${BOOST_PREFIX}") - set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") + set(Boost_INCLUDE_DIRS "${Boost_INCLUDE_DIR}") add_dependencies(boost_system_static boost_ep) add_dependencies(boost_filesystem_static boost_ep) add_dependencies(boost_serialization_static boost_ep) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 0383e00b53..17d5bcadef 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -6,6 +6,7 @@ server_config: db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path + db_slave_path: # secondry data storage path, split by semicolon # URI format: dialect://username:password@host:port/database # All parts except dialect are optional, but you MUST include the delimiters @@ -15,6 +16,8 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day + maximum_memory: 4 # maximum memory allowed, default: 4, unit: GB, should be at least 1 GB. + # the sum of maximum_memory and cpu_cache_capacity should be less than total memory metric_config: is_startup: off # if monitoring start: on, off @@ -33,4 +36,7 @@ cache_config: # cache configure cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory engine_config: - nprobe: 10 \ No newline at end of file + nprobe: 10 + nlist: 16384 + use_blas_threshold: 20 + metric_type: L2 #L2 or Inner Product diff --git a/cpp/coverage.sh b/cpp/coverage.sh index 7a48e5d451..8a899931a0 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -13,6 +13,27 @@ DIR_LCOV_OUTPUT="lcov_out" DIR_GCNO="cmake_build" DIR_UNITTEST="milvus/bin" + +MYSQL_USER_NAME=root +MYSQL_PASSWORD=Fantast1c +MYSQL_HOST='192.168.1.194' +MYSQL_PORT='3306' + +MYSQL_DB_NAME=milvus_`date +%s%N` + +function mysql_exc() +{ + cmd=$1 + mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}" + if [ $? -ne 0 ]; then + echo "mysql $cmd run failed" + fi +} + +mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};" +mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';" +mysql_exc "FLUSH PRIVILEGES;" +mysql_exc "USE ${MYSQL_DB_NAME};" MYSQL_USER_NAME=root MYSQL_PASSWORD=Fantast1c diff --git a/cpp/src/db/Constants.h b/cpp/src/db/Constants.h new file mode 100644 index 0000000000..055b10ca9a --- /dev/null +++ b/cpp/src/db/Constants.h @@ -0,0 +1,23 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ +#pragma once + +namespace zilliz { +namespace milvus { +namespace engine { + +const size_t K = 1024UL; +const size_t M = K * K; +const size_t G = K * M; +const size_t T = K * G; + +const size_t MAX_TABLE_FILE_MEM = 128 * M; + +const int VECTOR_TYPE_SIZE = sizeof(float); + +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index d1198b70df..a416312485 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -44,6 +44,8 @@ public: virtual Status Size(uint64_t& result) = 0; + virtual Status BuildIndex(const std::string& table_id) = 0; + virtual Status DropAll() = 0; DB() = default; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 0a1e8651e1..dca0f2a2ed 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -87,8 +87,7 @@ DBImpl::DBImpl(const Options& options) compact_thread_pool_(1, 1), index_thread_pool_(1, 1) { meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode); - mem_mgr_ = std::make_shared(meta_ptr_, options_); - // mem_mgr_ = (MemManagerPtr)(new MemManager(meta_ptr_, options_)); + mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_); if (options.mode != Options::MODE::READ_ONLY) { StartTimerTasks(); } @@ -408,10 +407,10 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->CleanUpFilesWithTTL(ttl); } -void DBImpl::StartBuildIndexTask() { +void DBImpl::StartBuildIndexTask(bool force) { static uint64_t index_clock_tick = 0; index_clock_tick++; - if(index_clock_tick%INDEX_ACTION_INTERVAL != 0) { + if(!force && (index_clock_tick%INDEX_ACTION_INTERVAL != 0)) { return; } @@ -430,6 +429,23 @@ void DBImpl::StartBuildIndexTask() { } } +Status DBImpl::BuildIndex(const std::string& table_id) { + bool has = false; + meta_ptr_->HasNonIndexFiles(table_id, has); + int times = 1; + + while (has) { + ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; + meta_ptr_->UpdateTableFilesToIndex(table_id); + /* StartBuildIndexTask(true); */ + std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100))); + meta_ptr_->HasNonIndexFiles(table_id, has); + times++; + } + return Status::OK(); + /* return BuildIndexByTable(table_id); */ +} + Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { ExecutionEnginePtr to_index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); if(to_index == nullptr) { @@ -444,6 +460,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { meta::TableFileSchema table_file; table_file.table_id_ = file.table_id_; table_file.date_ = file.date_; + table_file.file_type_ = meta::TableFileSchema::INDEX; //for multi-db-path, distribute index file averagely to each path Status status = meta_ptr_->CreateTableFile(table_file); if (!status.ok()) { return status; @@ -469,7 +486,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //step 6: update meta table_file.file_type_ = meta::TableFileSchema::INDEX; - table_file.size_ = index->Size(); + table_file.size_ = index->PhysicalSize(); auto to_remove = file; to_remove.file_type_ = meta::TableFileSchema::TO_DELETE; @@ -485,13 +502,35 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //index->Cache(); } catch (std::exception& ex) { - return Status::Error("Build index encounter exception", ex.what()); + std::string msg = "Build index encounter exception" + std::string(ex.what()); + ENGINE_LOG_ERROR << msg; + return Status::Error(msg); } return Status::OK(); } +Status DBImpl::BuildIndexByTable(const std::string& table_id) { + std::unique_lock lock(build_index_mutex_); + meta::TableFilesSchema to_index_files; + meta_ptr_->FilesToIndex(to_index_files); + + Status status; + + for (auto& file : to_index_files) { + status = BuildIndex(file); + if (!status.ok()) { + ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); + return status; + } + ENGINE_LOG_DEBUG << "Sync building index for " << file.id_ << " passed"; + } + + return status; +} + void DBImpl::BackgroundBuildIndex() { + std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; meta_ptr_->FilesToIndex(to_index_files); Status status; diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 9dcd174f8b..1309154a3c 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -9,6 +9,7 @@ #include "MemManager.h" #include "Types.h" #include "utils/ThreadPool.h" +#include "MemManagerAbstract.h" #include #include @@ -33,7 +34,6 @@ class Meta; class DBImpl : public DB { public: using MetaPtr = meta::Meta::Ptr; - using MemManagerPtr = typename MemManager::Ptr; explicit DBImpl(const Options &options); @@ -82,6 +82,8 @@ class DBImpl : public DB { Status Size(uint64_t &result) override; + Status BuildIndex(const std::string& table_id) override; + ~DBImpl() override; private: @@ -107,9 +109,11 @@ class DBImpl : public DB { Status BackgroundMergeFiles(const std::string &table_id); void BackgroundCompaction(std::set table_ids); - void StartBuildIndexTask(); + void StartBuildIndexTask(bool force=false); void BackgroundBuildIndex(); + Status + BuildIndexByTable(const std::string& table_id); Status BuildIndex(const meta::TableFileSchema &); @@ -123,7 +127,7 @@ class DBImpl : public DB { std::thread bg_timer_thread_; MetaPtr meta_ptr_; - MemManagerPtr mem_mgr_; + MemManagerAbstractPtr mem_mgr_; server::ThreadPool compact_thread_pool_; std::list> compact_thread_results_; @@ -132,6 +136,8 @@ class DBImpl : public DB { server::ThreadPool index_thread_pool_; std::list> index_thread_results_; + std::mutex build_index_mutex_; + }; // DBImpl diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index d13899dca0..fc67c6b360 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -83,26 +83,6 @@ using ConnectorT = decltype(StoragePrototype("")); static std::unique_ptr ConnectorPtr; using ConditionT = decltype(c(&TableFileSchema::id_) == 1UL); -std::string DBMetaImpl::GetTablePath(const std::string &table_id) { - return options_.path + "/tables/" + table_id; -} - -std::string DBMetaImpl::GetTableDatePartitionPath(const std::string &table_id, DateT &date) { - std::stringstream ss; - ss << GetTablePath(table_id) << "/" << date; - return ss.str(); -} - -void DBMetaImpl::GetTableFilePath(TableFileSchema &group_file) { - if (group_file.date_ == EmptyDate) { - group_file.date_ = Meta::GetDate(); - } - std::stringstream ss; - ss << GetTableDatePartitionPath(group_file.table_id_, group_file.date_) - << "/" << group_file.file_id_; - group_file.location_ = ss.str(); -} - Status DBMetaImpl::NextTableId(std::string &table_id) { std::stringstream ss; SimpleIDGenerator g; @@ -196,7 +176,8 @@ Status DBMetaImpl::CreateTable(TableSchema &table_schema) { if(TableSchema::TO_DELETE == std::get<0>(table[0])) { return Status::Error("Table already exists and it is in delete state, please wait a second"); } else { - return Status::OK();//table already exists, no error + // Change from no error to already exist. + return Status::AlreadyExist("Table already exists"); } } } @@ -212,15 +193,7 @@ Status DBMetaImpl::CreateTable(TableSchema &table_schema) { return Status::DBTransactionError("Add Table Error"); } - auto table_path = GetTablePath(table_schema.table_id_); - table_schema.location_ = table_path; - if (!boost::filesystem::is_directory(table_path)) { - auto ret = boost::filesystem::create_directories(table_path); - if (!ret) { - ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; - return Status::Error("Failed to create table path"); - } - } + return utils::CreateTablePath(options_, table_schema.table_id_); } catch (std::exception &e) { return HandleException("Encounter exception when create table", e); @@ -306,9 +279,6 @@ Status DBMetaImpl::DescribeTable(TableSchema &table_schema) { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); } - auto table_path = GetTablePath(table_schema.table_id_); - table_schema.location_ = table_path; - } catch (std::exception &e) { return HandleException("Encounter exception when describe table", e); } @@ -316,6 +286,28 @@ Status DBMetaImpl::DescribeTable(TableSchema &table_schema) { return Status::OK(); } +Status DBMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { + has = false; + try { + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_), + where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW + or + c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX) + and c(&TableFileSchema::table_id_) == table_id + )); + + if (selected.size() >= 1) { + has = true; + } else { + has = false; + } + + } catch (std::exception &e) { + return HandleException("Encounter exception when check non index files", e); + } + return Status::OK(); +} + Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { has_or_not = false; @@ -388,20 +380,11 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) { file_schema.created_on_ = utils::GetMicroSecTimeStamp(); file_schema.updated_time_ = file_schema.created_on_; file_schema.engine_type_ = table_schema.engine_type_; - GetTableFilePath(file_schema); auto id = ConnectorPtr->insert(file_schema); file_schema.id_ = id; - auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_); - - if (!boost::filesystem::is_directory(partition_path)) { - auto ret = boost::filesystem::create_directory(partition_path); - if (!ret) { - ENGINE_LOG_ERROR << "Create directory " << partition_path << " Error"; - return Status::DBTransactionError("Failed to create partition directory"); - } - } + return utils::CreateTableFilePath(options_, file_schema); } catch (std::exception& ex) { return HandleException("Encounter exception when create table file", ex); @@ -438,7 +421,7 @@ Status DBMetaImpl::FilesToIndex(TableFilesSchema &files) { table_file.date_ = std::get<5>(file); table_file.engine_type_ = std::get<6>(file); - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); auto groupItr = groups.find(table_file.table_id_); if (groupItr == groups.end()) { TableSchema table_schema; @@ -501,7 +484,7 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id, table_file.date_ = std::get<5>(file); table_file.engine_type_ = std::get<6>(file); table_file.dimension_ = table_schema.dimension_; - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { files[table_file.date_] = TableFilesSchema(); @@ -543,7 +526,7 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id, table_file.date_ = std::get<5>(file); table_file.engine_type_ = std::get<6>(file); table_file.dimension_ = table_schema.dimension_; - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { files[table_file.date_] = TableFilesSchema(); @@ -593,7 +576,7 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id, table_file.size_ = std::get<4>(file); table_file.date_ = std::get<5>(file); table_file.dimension_ = table_schema.dimension_; - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { files[table_file.date_] = TableFilesSchema(); @@ -639,7 +622,7 @@ Status DBMetaImpl::GetTableFiles(const std::string& table_id, file_schema.date_ = std::get<4>(file); file_schema.engine_type_ = std::get<5>(file); file_schema.dimension_ = table_schema.dimension_; - GetTableFilePath(file_schema); + utils::GetTableFilePath(options_, file_schema); table_files.emplace_back(file_schema); } @@ -791,6 +774,23 @@ Status DBMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { return Status::OK(); } +Status DBMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { + try { + ConnectorPtr->update_all( + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_INDEX + ), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW + )); + } catch (std::exception &e) { + return HandleException("Encounter exception when update table files to to_index", e); + } + + return Status::OK(); +} + Status DBMetaImpl::UpdateTableFiles(TableFilesSchema &files) { try { MetricCollector metric; @@ -855,10 +855,9 @@ Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { table_file.table_id_ = std::get<1>(file); table_file.file_id_ = std::get<2>(file); table_file.date_ = std::get<3>(file); - GetTableFilePath(table_file); + utils::DeleteTableFilePath(options_, table_file); ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " << table_file.location_ << std::endl; - boost::filesystem::remove(table_file.location_); ConnectorPtr->remove(table_file.id_); } @@ -882,10 +881,7 @@ Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto commited = ConnectorPtr->transaction([&]() mutable { for (auto &table : tables) { - auto table_path = GetTablePath(std::get<1>(table)); - - ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; - boost::filesystem::remove_all(table_path); + utils::DeleteTablePath(options_, std::get<1>(table)); ConnectorPtr->remove(std::get<0>(table)); } diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 1d5f400622..98a42c000d 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -35,6 +35,10 @@ public: const std::vector& ids, TableFilesSchema& table_files) override; + virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override; + + virtual Status UpdateTableFilesToIndex(const std::string& table_id) override; + virtual Status UpdateTableFile(TableFileSchema& file_schema) override; virtual Status UpdateTableFiles(TableFilesSchema& files) override; @@ -66,9 +70,6 @@ private: Status NextFileId(std::string& file_id); Status NextTableId(std::string& table_id); Status DiscardFiles(long to_discard_size); - std::string GetTablePath(const std::string& table_id); - std::string GetTableDatePartitionPath(const std::string& table_id, DateT& date); - void GetTableFilePath(TableFileSchema& group_file); Status Initialize(); const DBMetaOptions options_; diff --git a/cpp/src/db/EngineFactory.cpp b/cpp/src/db/EngineFactory.cpp index bacce70ce4..2ca4b24aa1 100644 --- a/cpp/src/db/EngineFactory.cpp +++ b/cpp/src/db/EngineFactory.cpp @@ -22,13 +22,19 @@ EngineFactory::Build(uint16_t dimension, switch (type) { case EngineType::FAISS_IDMAP: { execution_engine_ptr = - ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat")); + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IDMAP, "IDMap,Flat")); break; } case EngineType::FAISS_IVFFLAT: { execution_engine_ptr = - ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat")); + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IVF, "IDMap,Flat")); + break; + } + + case EngineType::FAISS_IVFSQ8: { + execution_engine_ptr = + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IVFSQ8, "IDMap,Flat")); break; } diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/ExecutionEngine.h index f8c05f6f9d..4503e55650 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/ExecutionEngine.h @@ -18,6 +18,7 @@ enum class EngineType { INVALID = 0, FAISS_IDMAP = 1, FAISS_IVFFLAT, + FAISS_IVFSQ8, }; class ExecutionEngine { diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 4b24bd3a1c..442dca2974 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -3,10 +3,14 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// -#include + #include "Factories.h" #include "DBImpl.h" +#include "MemManager.h" +#include "NewMemManager.h" +#include "Exception.h" +#include #include #include #include @@ -14,7 +18,9 @@ #include #include #include -#include "Exception.h" +#include +#include +#include namespace zilliz { namespace milvus { @@ -72,17 +78,14 @@ std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOp if (dialect.find("mysql") != std::string::npos) { ENGINE_LOG_INFO << "Using MySQL"; return std::make_shared(meta::MySQLMetaImpl(metaOptions, mode)); - } - else if (dialect.find("sqlite") != std::string::npos) { + } else if (dialect.find("sqlite") != std::string::npos) { ENGINE_LOG_INFO << "Using SQLite"; return std::make_shared(meta::DBMetaImpl(metaOptions)); - } - else { + } else { ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect; throw InvalidArgumentException("URI dialect is not mysql / sqlite"); } - } - else { + } else { ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri; throw InvalidArgumentException("Wrong URI format "); } @@ -98,6 +101,21 @@ DB* DBFactory::Build(const Options& options) { return new DBImpl(options); } +MemManagerAbstractPtr MemManagerFactory::Build(const std::shared_ptr& meta, + const Options& options) { + if (const char* env = getenv("MILVUS_USE_OLD_MEM_MANAGER")) { + std::string env_str = env; + std::transform(env_str.begin(), env_str.end(), env_str.begin(), ::toupper); + if (env_str == "ON") { + return std::make_shared(meta, options); + } + else { + return std::make_shared(meta, options); + } + } + return std::make_shared(meta, options); +} + } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 889922b17a..8b6e7b100f 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -10,16 +10,18 @@ #include "MySQLMetaImpl.h" #include "Options.h" #include "ExecutionEngine.h" +#include "MemManagerAbstract.h" #include #include + namespace zilliz { namespace milvus { namespace engine { struct DBMetaOptionsFactory { - static DBMetaOptions Build(const std::string& path = ""); + static DBMetaOptions Build(const std::string &path = ""); }; struct OptionsFactory { @@ -28,12 +30,16 @@ struct OptionsFactory { struct DBMetaImplFactory { static std::shared_ptr Build(); - static std::shared_ptr Build(const DBMetaOptions& metaOptions, const int& mode); + static std::shared_ptr Build(const DBMetaOptions &metaOptions, const int &mode); }; struct DBFactory { static std::shared_ptr Build(); - static DB* Build(const Options&); + static DB *Build(const Options &); +}; + +struct MemManagerFactory { + static MemManagerAbstractPtr Build(const std::shared_ptr &meta, const Options &options); }; } // namespace engine diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 20bd530e78..d2cb835364 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -5,6 +5,7 @@ ******************************************************************************/ #include "FaissExecutionEngine.h" #include "Log.h" +#include "utils/CommonUtil.h" #include #include @@ -21,15 +22,25 @@ namespace zilliz { namespace milvus { namespace engine { +namespace { +std::string GetMetricType() { + server::ServerConfig &config = server::ServerConfig::GetInstance(); + server::ConfigNode engine_config = config.GetConfig(server::CONFIG_ENGINE); + return engine_config.GetValue(server::CONFIG_METRICTYPE, "L2"); +} +} FaissExecutionEngine::FaissExecutionEngine(uint16_t dimension, const std::string& location, const std::string& build_index_type, const std::string& raw_index_type) - : pIndex_(faiss::index_factory(dimension, raw_index_type.c_str())), - location_(location), + : location_(location), build_index_type_(build_index_type), raw_index_type_(raw_index_type) { + + std::string metric_type = GetMetricType(); + faiss::MetricType faiss_metric_type = (metric_type == "L2") ? faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + pIndex_.reset(faiss::index_factory(dimension, raw_index_type.c_str(), faiss_metric_type)); } FaissExecutionEngine::FaissExecutionEngine(std::shared_ptr index, @@ -60,7 +71,7 @@ size_t FaissExecutionEngine::Dimension() const { } size_t FaissExecutionEngine::PhysicalSize() const { - return (size_t)(Count() * pIndex_->d)*sizeof(float); + return server::CommonUtil::GetFileSize(location_); } Status FaissExecutionEngine::Serialize() { @@ -118,6 +129,7 @@ FaissExecutionEngine::BuildIndex(const std::string& location) { auto opd = std::make_shared(); opd->d = pIndex_->d; opd->index_type = build_index_type_; + opd->metric_type = GetMetricType(); IndexBuilderPtr pBuilder = GetIndexBuilder(opd); auto from_index = dynamic_cast(pIndex_.get()); @@ -161,14 +173,16 @@ Status FaissExecutionEngine::Cache() { Status FaissExecutionEngine::Init() { - if(build_index_type_ == "IVF") { + if(build_index_type_ == BUILD_INDEX_TYPE_IVF || + build_index_type_ == BUILD_INDEX_TYPE_IVFSQ8) { using namespace zilliz::milvus::server; ServerConfig &config = ServerConfig::GetInstance(); ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000); + nlist_ = engine_config.GetInt32Value(CONFIG_NLIST,16384); - } else if(build_index_type_ == "IDMap") { + } else if(build_index_type_ == BUILD_INDEX_TYPE_IDMAP) { ; } else { return Status::Error("Wrong index type: ", build_index_type_); diff --git a/cpp/src/db/FaissExecutionEngine.h b/cpp/src/db/FaissExecutionEngine.h index f9f37ad978..7005061ff6 100644 --- a/cpp/src/db/FaissExecutionEngine.h +++ b/cpp/src/db/FaissExecutionEngine.h @@ -15,6 +15,9 @@ namespace zilliz { namespace milvus { namespace engine { +const static std::string BUILD_INDEX_TYPE_IDMAP = "IDMap"; +const static std::string BUILD_INDEX_TYPE_IVF = "IVF"; +const static std::string BUILD_INDEX_TYPE_IVFSQ8 = "IVFSQ8"; class FaissExecutionEngine : public ExecutionEngine { public: @@ -65,6 +68,7 @@ protected: std::string raw_index_type_; size_t nprobe_ = 0; + size_t nlist_ = 0; }; diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index e36b0c45ba..dbf0703173 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -8,28 +8,30 @@ #include "MetaConsts.h" #include "EngineFactory.h" #include "metrics/Metrics.h" +#include "Log.h" #include #include #include #include + namespace zilliz { namespace milvus { namespace engine { -MemVectors::MemVectors(const std::shared_ptr& meta_ptr, - const meta::TableFileSchema& schema, const Options& options) - : meta_(meta_ptr), - options_(options), - schema_(schema), - id_generator_(new SimpleIDGenerator()), - active_engine_(EngineFactory::Build(schema_.dimension_, schema_.location_, (EngineType)schema_.engine_type_)) { +MemVectors::MemVectors(const std::shared_ptr &meta_ptr, + const meta::TableFileSchema &schema, const Options &options) + : meta_(meta_ptr), + options_(options), + schema_(schema), + id_generator_(new SimpleIDGenerator()), + active_engine_(EngineFactory::Build(schema_.dimension_, schema_.location_, (EngineType) schema_.engine_type_)) { } -Status MemVectors::Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { - if(active_engine_ == nullptr) { +Status MemVectors::Add(size_t n_, const float *vectors_, IDNumbers &vector_ids_) { + if (active_engine_ == nullptr) { return Status::Error("index engine is null"); } @@ -38,13 +40,15 @@ Status MemVectors::Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) Status status = active_engine_->AddWithIds(n_, vectors_, vector_ids_.data()); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), static_cast(schema_.dimension_), total_time); + server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), + static_cast(schema_.dimension_), + total_time); return status; } size_t MemVectors::RowCount() const { - if(active_engine_ == nullptr) { + if (active_engine_ == nullptr) { return 0; } @@ -52,15 +56,15 @@ size_t MemVectors::RowCount() const { } size_t MemVectors::Size() const { - if(active_engine_ == nullptr) { + if (active_engine_ == nullptr) { return 0; } return active_engine_->Size(); } -Status MemVectors::Serialize(std::string& table_id) { - if(active_engine_ == nullptr) { +Status MemVectors::Serialize(std::string &table_id) { + if (active_engine_ == nullptr) { return Status::Error("index engine is null"); } @@ -72,15 +76,16 @@ Status MemVectors::Serialize(std::string& table_id) { auto total_time = METRICS_MICROSECONDS(start_time, end_time); schema_.size_ = size; - server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet(size/total_time); + server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet(size / total_time); schema_.file_type_ = (size >= options_.index_trigger_size) ? - meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; auto status = meta_->UpdateTableFile(schema_); LOG(DEBUG) << "New " << ((schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") - << " file " << schema_.file_id_ << " of size " << (double)(active_engine_->Size()) / (double)meta::M << " M"; + << " file " << schema_.file_id_ << " of size " << (double) (active_engine_->Size()) / (double) meta::M + << " M"; active_engine_->Cache(); @@ -98,7 +103,7 @@ MemVectors::~MemVectors() { * MemManager */ MemManager::MemVectorsPtr MemManager::GetMemByTable( - const std::string& table_id) { + const std::string &table_id) { auto memIt = mem_id_map_.find(table_id); if (memIt != mem_id_map_.end()) { return memIt->second; @@ -115,26 +120,31 @@ MemManager::MemVectorsPtr MemManager::GetMemByTable( return mem_id_map_[table_id]; } -Status MemManager::InsertVectors(const std::string& table_id_, - size_t n_, - const float* vectors_, - IDNumbers& vector_ids_) { +Status MemManager::InsertVectors(const std::string &table_id_, + size_t n_, + const float *vectors_, + IDNumbers &vector_ids_) { + + LOG(DEBUG) << "MemManager::InsertVectors: mutable mem = " << GetCurrentMutableMem() << + ", immutable mem = " << GetCurrentImmutableMem() << ", total mem = " << GetCurrentMem(); + std::unique_lock lock(mutex_); return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); } -Status MemManager::InsertVectorsNoLock(const std::string& table_id, - size_t n, - const float* vectors, - IDNumbers& vector_ids) { +Status MemManager::InsertVectorsNoLock(const std::string &table_id, + size_t n, + const float *vectors, + IDNumbers &vector_ids) { + MemVectorsPtr mem = GetMemByTable(table_id); if (mem == nullptr) { return Status::NotFound("Group " + table_id + " not found!"); } //makesure each file size less than index_trigger_size - if(mem->Size() > options_.index_trigger_size) { + if (mem->Size() > options_.index_trigger_size) { std::unique_lock lock(serialization_mtx_); immu_mem_list_.push_back(mem); mem_id_map_.erase(table_id); @@ -147,8 +157,8 @@ Status MemManager::InsertVectorsNoLock(const std::string& table_id, Status MemManager::ToImmutable() { std::unique_lock lock(mutex_); MemIdMap temp_map; - for (auto& kv: mem_id_map_) { - if(kv.second->RowCount() == 0) { + for (auto &kv: mem_id_map_) { + if (kv.second->RowCount() == 0) { temp_map.insert(kv); continue;//empty vector, no need to serialize } @@ -159,12 +169,12 @@ Status MemManager::ToImmutable() { return Status::OK(); } -Status MemManager::Serialize(std::set& table_ids) { +Status MemManager::Serialize(std::set &table_ids) { ToImmutable(); std::unique_lock lock(serialization_mtx_); std::string table_id; table_ids.clear(); - for (auto& mem : immu_mem_list_) { + for (auto &mem : immu_mem_list_) { mem->Serialize(table_id); table_ids.insert(table_id); } @@ -172,7 +182,7 @@ Status MemManager::Serialize(std::set& table_ids) { return Status::OK(); } -Status MemManager::EraseMemVector(const std::string& table_id) { +Status MemManager::EraseMemVector(const std::string &table_id) { {//erase MemVector from rapid-insert cache std::unique_lock lock(mutex_); mem_id_map_.erase(table_id); @@ -181,8 +191,8 @@ Status MemManager::EraseMemVector(const std::string& table_id) { {//erase MemVector from serialize cache std::unique_lock lock(serialization_mtx_); MemList temp_list; - for (auto& mem : immu_mem_list_) { - if(mem->TableId() != table_id) { + for (auto &mem : immu_mem_list_) { + if (mem->TableId() != table_id) { temp_list.push_back(mem); } } @@ -192,6 +202,26 @@ Status MemManager::EraseMemVector(const std::string& table_id) { return Status::OK(); } +size_t MemManager::GetCurrentMutableMem() { + size_t totalMem = 0; + for (auto &kv : mem_id_map_) { + auto memVector = kv.second; + totalMem += memVector->Size(); + } + return totalMem; +} + +size_t MemManager::GetCurrentImmutableMem() { + size_t totalMem = 0; + for (auto &memVector : immu_mem_list_) { + totalMem += memVector->Size(); + } + return totalMem; +} + +size_t MemManager::GetCurrentMem() { + return GetCurrentMutableMem() + GetCurrentImmutableMem(); +} } // namespace engine } // namespace milvus diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index 0ce88d504d..4dc20581df 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -9,6 +9,7 @@ #include "IDGenerator.h" #include "Status.h" #include "Meta.h" +#include "MemManagerAbstract.h" #include #include @@ -17,72 +18,79 @@ #include #include + namespace zilliz { namespace milvus { namespace engine { namespace meta { - class Meta; +class Meta; } class MemVectors { -public: + public: using MetaPtr = meta::Meta::Ptr; using Ptr = std::shared_ptr; - explicit MemVectors(const std::shared_ptr&, - const meta::TableFileSchema&, const Options&); + explicit MemVectors(const std::shared_ptr &, + const meta::TableFileSchema &, const Options &); - Status Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_); + Status Add(size_t n_, const float *vectors_, IDNumbers &vector_ids_); size_t RowCount() const; size_t Size() const; - Status Serialize(std::string& table_id); + Status Serialize(std::string &table_id); ~MemVectors(); - const std::string& Location() const { return schema_.location_; } + const std::string &Location() const { return schema_.location_; } std::string TableId() const { return schema_.table_id_; } -private: + private: MemVectors() = delete; - MemVectors(const MemVectors&) = delete; - MemVectors& operator=(const MemVectors&) = delete; + MemVectors(const MemVectors &) = delete; + MemVectors &operator=(const MemVectors &) = delete; MetaPtr meta_; Options options_; meta::TableFileSchema schema_; - IDGenerator* id_generator_; + IDGenerator *id_generator_; ExecutionEnginePtr active_engine_; }; // MemVectors -class MemManager { -public: +class MemManager : public MemManagerAbstract { + public: using MetaPtr = meta::Meta::Ptr; using MemVectorsPtr = typename MemVectors::Ptr; using Ptr = std::shared_ptr; - MemManager(const std::shared_ptr& meta, const Options& options) + MemManager(const std::shared_ptr &meta, const Options &options) : meta_(meta), options_(options) {} - MemVectorsPtr GetMemByTable(const std::string& table_id); + Status InsertVectors(const std::string &table_id, + size_t n, const float *vectors, IDNumbers &vector_ids) override; - Status InsertVectors(const std::string& table_id, - size_t n, const float* vectors, IDNumbers& vector_ids); + Status Serialize(std::set &table_ids) override; - Status Serialize(std::set& table_ids); + Status EraseMemVector(const std::string &table_id) override; - Status EraseMemVector(const std::string& table_id); + size_t GetCurrentMutableMem() override; -private: - Status InsertVectorsNoLock(const std::string& table_id, - size_t n, const float* vectors, IDNumbers& vector_ids); + size_t GetCurrentImmutableMem() override; + + size_t GetCurrentMem() override; + + private: + MemVectorsPtr GetMemByTable(const std::string &table_id); + + Status InsertVectorsNoLock(const std::string &table_id, + size_t n, const float *vectors, IDNumbers &vector_ids); Status ToImmutable(); using MemIdMap = std::map; diff --git a/cpp/src/db/MemManagerAbstract.h b/cpp/src/db/MemManagerAbstract.h new file mode 100644 index 0000000000..943c454e46 --- /dev/null +++ b/cpp/src/db/MemManagerAbstract.h @@ -0,0 +1,32 @@ +#pragma once + +#include + + +namespace zilliz { +namespace milvus { +namespace engine { + +class MemManagerAbstract { + public: + + virtual Status InsertVectors(const std::string &table_id, + size_t n, const float *vectors, IDNumbers &vector_ids) = 0; + + virtual Status Serialize(std::set &table_ids) = 0; + + virtual Status EraseMemVector(const std::string &table_id) = 0; + + virtual size_t GetCurrentMutableMem() = 0; + + virtual size_t GetCurrentImmutableMem() = 0; + + virtual size_t GetCurrentMem() = 0; + +}; // MemManagerAbstract + +using MemManagerAbstractPtr = std::shared_ptr; + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MemTable.cpp b/cpp/src/db/MemTable.cpp new file mode 100644 index 0000000000..e05aa058ac --- /dev/null +++ b/cpp/src/db/MemTable.cpp @@ -0,0 +1,88 @@ +#include "MemTable.h" +#include "Log.h" + + +namespace zilliz { +namespace milvus { +namespace engine { + +MemTable::MemTable(const std::string &table_id, + const std::shared_ptr &meta, + const Options &options) : + table_id_(table_id), + meta_(meta), + options_(options) { + +} + +Status MemTable::Add(VectorSource::Ptr &source) { + + while (!source->AllAdded()) { + + MemTableFile::Ptr current_mem_table_file; + if (!mem_table_file_list_.empty()) { + current_mem_table_file = mem_table_file_list_.back(); + } + + Status status; + if (mem_table_file_list_.empty() || current_mem_table_file->IsFull()) { + MemTableFile::Ptr new_mem_table_file = std::make_shared(table_id_, meta_, options_); + status = new_mem_table_file->Add(source); + if (status.ok()) { + mem_table_file_list_.emplace_back(new_mem_table_file); + } + } else { + status = current_mem_table_file->Add(source); + } + + if (!status.ok()) { + std::string err_msg = "MemTable::Add failed: " + status.ToString(); + ENGINE_LOG_ERROR << err_msg; + return Status::Error(err_msg); + } + } + return Status::OK(); +} + +void MemTable::GetCurrentMemTableFile(MemTableFile::Ptr &mem_table_file) { + mem_table_file = mem_table_file_list_.back(); +} + +size_t MemTable::GetTableFileCount() { + return mem_table_file_list_.size(); +} + +Status MemTable::Serialize() { + for (auto mem_table_file = mem_table_file_list_.begin(); mem_table_file != mem_table_file_list_.end();) { + auto status = (*mem_table_file)->Serialize(); + if (!status.ok()) { + std::string err_msg = "MemTable::Serialize failed: " + status.ToString(); + ENGINE_LOG_ERROR << err_msg; + return Status::Error(err_msg); + } + std::lock_guard lock(mutex_); + mem_table_file = mem_table_file_list_.erase(mem_table_file); + } + return Status::OK(); +} + +bool MemTable::Empty() { + return mem_table_file_list_.empty(); +} + +const std::string &MemTable::GetTableId() const { + return table_id_; +} + +size_t MemTable::GetCurrentMem() { + std::lock_guard lock(mutex_); + size_t total_mem = 0; + for (auto &mem_table_file : mem_table_file_list_) { + total_mem += mem_table_file->GetCurrentMem(); + } + return total_mem; +} + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MemTable.h b/cpp/src/db/MemTable.h new file mode 100644 index 0000000000..198fcc228a --- /dev/null +++ b/cpp/src/db/MemTable.h @@ -0,0 +1,53 @@ +#pragma once + +#include "Status.h" +#include "MemTableFile.h" +#include "VectorSource.h" + +#include + + +namespace zilliz { +namespace milvus { +namespace engine { + +class MemTable { + + public: + + using Ptr = std::shared_ptr; + using MemTableFileList = std::vector; + using MetaPtr = meta::Meta::Ptr; + + MemTable(const std::string &table_id, const std::shared_ptr &meta, const Options &options); + + Status Add(VectorSource::Ptr &source); + + void GetCurrentMemTableFile(MemTableFile::Ptr &mem_table_file); + + size_t GetTableFileCount(); + + Status Serialize(); + + bool Empty(); + + const std::string &GetTableId() const; + + size_t GetCurrentMem(); + + private: + const std::string table_id_; + + MemTableFileList mem_table_file_list_; + + MetaPtr meta_; + + Options options_; + + std::mutex mutex_; + +}; //MemTable + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MemTableFile.cpp b/cpp/src/db/MemTableFile.cpp new file mode 100644 index 0000000000..649a680cf3 --- /dev/null +++ b/cpp/src/db/MemTableFile.cpp @@ -0,0 +1,108 @@ +#include "MemTableFile.h" +#include "Constants.h" +#include "Log.h" +#include "EngineFactory.h" +#include "metrics/Metrics.h" + +#include + + +namespace zilliz { +namespace milvus { +namespace engine { + +MemTableFile::MemTableFile(const std::string &table_id, + const std::shared_ptr &meta, + const Options &options) : + table_id_(table_id), + meta_(meta), + options_(options) { + + current_mem_ = 0; + auto status = CreateTableFile(); + if (status.ok()) { + execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_, + table_file_schema_.location_, + (EngineType) table_file_schema_.engine_type_); + } +} + +Status MemTableFile::CreateTableFile() { + + meta::TableFileSchema table_file_schema; + table_file_schema.table_id_ = table_id_; + auto status = meta_->CreateTableFile(table_file_schema); + if (status.ok()) { + table_file_schema_ = table_file_schema; + } else { + std::string err_msg = "MemTableFile::CreateTableFile failed: " + status.ToString(); + ENGINE_LOG_ERROR << err_msg; + } + return status; +} + +Status MemTableFile::Add(const VectorSource::Ptr &source) { + + if (table_file_schema_.dimension_ <= 0) { + std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + + std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; + ENGINE_LOG_ERROR << err_msg; + return Status::Error(err_msg); + } + + size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; + size_t mem_left = GetMemLeft(); + if (mem_left >= single_vector_mem_size) { + size_t num_vectors_to_add = std::ceil(mem_left / single_vector_mem_size); + size_t num_vectors_added; + auto status = source->Add(execution_engine_, table_file_schema_, num_vectors_to_add, num_vectors_added); + if (status.ok()) { + current_mem_ += (num_vectors_added * single_vector_mem_size); + } + return status; + } + return Status::OK(); +} + +size_t MemTableFile::GetCurrentMem() { + return current_mem_; +} + +size_t MemTableFile::GetMemLeft() { + return (MAX_TABLE_FILE_MEM - current_mem_); +} + +bool MemTableFile::IsFull() { + size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; + return (GetMemLeft() < single_vector_mem_size); +} + +Status MemTableFile::Serialize() { + + auto start_time = METRICS_NOW_TIME; + + auto size = GetCurrentMem(); + + execution_engine_->Serialize(); + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + table_file_schema_.size_ = size; + + server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double) size / total_time); + + table_file_schema_.file_type_ = (size >= options_.index_trigger_size) ? + meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + + auto status = meta_->UpdateTableFile(table_file_schema_); + + LOG(DEBUG) << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") + << " file " << table_file_schema_.file_id_ << " of size " << (double) size / (double) M << " M"; + + execution_engine_->Cache(); + + return status; +} + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MemTableFile.h b/cpp/src/db/MemTableFile.h new file mode 100644 index 0000000000..4d0011b362 --- /dev/null +++ b/cpp/src/db/MemTableFile.h @@ -0,0 +1,52 @@ +#pragma once + +#include "Status.h" +#include "Meta.h" +#include "VectorSource.h" +#include "ExecutionEngine.h" + + +namespace zilliz { +namespace milvus { +namespace engine { + +class MemTableFile { + + public: + + using Ptr = std::shared_ptr; + using MetaPtr = meta::Meta::Ptr; + + MemTableFile(const std::string &table_id, const std::shared_ptr &meta, const Options &options); + + Status Add(const VectorSource::Ptr &source); + + size_t GetCurrentMem(); + + size_t GetMemLeft(); + + bool IsFull(); + + Status Serialize(); + + private: + + Status CreateTableFile(); + + const std::string table_id_; + + meta::TableFileSchema table_file_schema_; + + MetaPtr meta_; + + Options options_; + + size_t current_mem_; + + ExecutionEnginePtr execution_engine_; + +}; //MemTableFile + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 6c6dd28bd0..1d83817f5d 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -39,6 +39,8 @@ public: const std::vector& ids, TableFilesSchema& table_files) = 0; + virtual Status UpdateTableFilesToIndex(const std::string& table_id) = 0; + virtual Status UpdateTableFile(TableFileSchema& file_schema) = 0; virtual Status UpdateTableFiles(TableFilesSchema& files) = 0; @@ -56,6 +58,8 @@ public: virtual Status FilesToIndex(TableFilesSchema&) = 0; + virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) = 0; + virtual Status CleanUp() = 0; virtual Status CleanUpFilesWithTTL(uint16_t) = 0; diff --git a/cpp/src/db/MetaTypes.h b/cpp/src/db/MetaTypes.h index b20d3a05d0..641712e9d5 100644 --- a/cpp/src/db/MetaTypes.h +++ b/cpp/src/db/MetaTypes.h @@ -31,7 +31,6 @@ struct TableSchema { int state_ = (int)NORMAL; size_t files_cnt_ = 0; uint16_t dimension_ = 0; - std::string location_; long created_on_ = 0; int engine_type_ = (int)EngineType::FAISS_IDMAP; bool store_raw_data_ = false; diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 8ba7f1d3b6..bd82e7e6a1 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -67,26 +67,6 @@ namespace meta { } - std::string MySQLMetaImpl::GetTablePath(const std::string &table_id) { - return options_.path + "/tables/" + table_id; - } - - std::string MySQLMetaImpl::GetTableDatePartitionPath(const std::string &table_id, DateT &date) { - std::stringstream ss; - ss << GetTablePath(table_id) << "/" << date; - return ss.str(); - } - - void MySQLMetaImpl::GetTableFilePath(TableFileSchema &group_file) { - if (group_file.date_ == EmptyDate) { - group_file.date_ = Meta::GetDate(); - } - std::stringstream ss; - ss << GetTableDatePartitionPath(group_file.table_id_, group_file.date_) - << "/" << group_file.file_id_; - group_file.location_ = ss.str(); - } - Status MySQLMetaImpl::NextTableId(std::string &table_id) { std::stringstream ss; SimpleIDGenerator g; @@ -412,15 +392,8 @@ namespace meta { // auto total_time = METRICS_MICROSECONDS(start_time, end_time); // server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - auto table_path = GetTablePath(table_schema.table_id_); - table_schema.location_ = table_path; - if (!boost::filesystem::is_directory(table_path)) { - auto ret = boost::filesystem::create_directories(table_path); - if (!ret) { - ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; - return Status::Error("Failed to create table path"); - } - } + return utils::CreateTablePath(options_, table_schema.table_id_); + } catch (const BadQuery& er) { // Handle any query errors ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE" << ": " << er.what(); @@ -436,6 +409,11 @@ namespace meta { return Status::OK(); } + Status MySQLMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { + // TODO + return Status::OK(); + } + Status MySQLMetaImpl::DeleteTable(const std::string& table_id) { // std::lock_guard lock(mysql_mutex); @@ -583,9 +561,6 @@ namespace meta { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); } - auto table_path = GetTablePath(table_schema.table_id_); - table_schema.location_ = table_path; - } catch (const BadQuery& er) { // Handle any query errors ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); @@ -738,7 +713,7 @@ namespace meta { file_schema.created_on_ = utils::GetMicroSecTimeStamp(); file_schema.updated_time_ = file_schema.created_on_; file_schema.engine_type_ = table_schema.engine_type_; - GetTableFilePath(file_schema); + utils::GetTableFilePath(options_, file_schema); std::string id = "NULL"; //auto-increment std::string table_id = file_schema.table_id_; @@ -783,15 +758,7 @@ namespace meta { } } // Scoped Connection - auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_); - - if (!boost::filesystem::is_directory(partition_path)) { - auto ret = boost::filesystem::create_directory(partition_path); - if (!ret) { - ENGINE_LOG_ERROR << "Create directory " << partition_path << " Error"; - return Status::DBTransactionError("Failed to create partition directory"); - } - } + return utils::CreateTableFilePath(options_, file_schema); } catch (const BadQuery& er) { // Handle any query errors @@ -876,7 +843,7 @@ namespace meta { } table_file.dimension_ = groups[table_file.table_id_].dimension_; - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); files.push_back(table_file); } @@ -987,7 +954,7 @@ namespace meta { table_file.dimension_ = table_schema.dimension_; - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { @@ -1073,7 +1040,7 @@ namespace meta { table_file.dimension_ = table_schema.dimension_; - GetTableFilePath(table_file); + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { @@ -1168,7 +1135,7 @@ namespace meta { file_schema.dimension_ = table_schema.dimension_; - GetTableFilePath(file_schema); + utils::GetTableFilePath(options_, file_schema); table_files.emplace_back(file_schema); } @@ -1479,6 +1446,11 @@ namespace meta { return Status::OK(); } + Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { + // TODO + return Status::OK(); + } + Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { // std::lock_guard lock(mysql_mutex); @@ -1623,11 +1595,10 @@ namespace meta { table_file.date_ = resRow["date"]; - GetTableFilePath(table_file); + utils::DeleteTableFilePath(options_, table_file); ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " << table_file.location_ << std::endl; - boost::filesystem::remove(table_file.location_); idsToDelete.emplace_back(std::to_string(table_file.id_)); } @@ -1700,10 +1671,7 @@ namespace meta { std::string table_id; resRow["table_id"].to_string(table_id); - auto table_path = GetTablePath(table_id); - - ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; - boost::filesystem::remove_all(table_path); + utils::DeleteTablePath(options_, table_id); idsToDeleteSS << "id = " << std::to_string(id) << " OR "; } diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index 9ff8254b60..403d5b0b59 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -40,8 +40,12 @@ namespace meta { const std::vector& ids, TableFilesSchema& table_files) override; + virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override; + virtual Status UpdateTableFile(TableFileSchema& file_schema) override; + virtual Status UpdateTableFilesToIndex(const std::string& table_id) override; + virtual Status UpdateTableFiles(TableFilesSchema& files) override; virtual Status FilesToSearch(const std::string& table_id, @@ -71,9 +75,6 @@ namespace meta { Status NextFileId(std::string& file_id); Status NextTableId(std::string& table_id); Status DiscardFiles(long long to_discard_size); - std::string GetTablePath(const std::string& table_id); - std::string GetTableDatePartitionPath(const std::string& table_id, DateT& date); - void GetTableFilePath(TableFileSchema& group_file); Status Initialize(); const DBMetaOptions options_; diff --git a/cpp/src/db/NewMemManager.cpp b/cpp/src/db/NewMemManager.cpp new file mode 100644 index 0000000000..e4903d75a9 --- /dev/null +++ b/cpp/src/db/NewMemManager.cpp @@ -0,0 +1,126 @@ +#include "NewMemManager.h" +#include "VectorSource.h" +#include "Log.h" +#include "Constants.h" + +#include + + +namespace zilliz { +namespace milvus { +namespace engine { + +NewMemManager::MemTablePtr NewMemManager::GetMemByTable(const std::string &table_id) { + auto memIt = mem_id_map_.find(table_id); + if (memIt != mem_id_map_.end()) { + return memIt->second; + } + + mem_id_map_[table_id] = std::make_shared(table_id, meta_, options_); + return mem_id_map_[table_id]; +} + +Status NewMemManager::InsertVectors(const std::string &table_id_, + size_t n_, + const float *vectors_, + IDNumbers &vector_ids_) { + + while (GetCurrentMem() > options_.maximum_memory) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + LOG(DEBUG) << "NewMemManager::InsertVectors: mutable mem = " << GetCurrentMutableMem() << + ", immutable mem = " << GetCurrentImmutableMem() << ", total mem = " << GetCurrentMem(); + + std::unique_lock lock(mutex_); + + return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); +} + +Status NewMemManager::InsertVectorsNoLock(const std::string &table_id, + size_t n, + const float *vectors, + IDNumbers &vector_ids) { + + MemTablePtr mem = GetMemByTable(table_id); + VectorSource::Ptr source = std::make_shared(n, vectors); + + auto status = mem->Add(source); + if (status.ok()) { + vector_ids = source->GetVectorIds(); + } + return status; +} + +Status NewMemManager::ToImmutable() { + std::unique_lock lock(mutex_); + MemIdMap temp_map; + for (auto &kv: mem_id_map_) { + if (kv.second->Empty()) { + //empty table, no need to serialize + temp_map.insert(kv); + } else { + immu_mem_list_.push_back(kv.second); + } + } + + mem_id_map_.swap(temp_map); + return Status::OK(); +} + +Status NewMemManager::Serialize(std::set &table_ids) { + ToImmutable(); + std::unique_lock lock(serialization_mtx_); + table_ids.clear(); + for (auto &mem : immu_mem_list_) { + mem->Serialize(); + table_ids.insert(mem->GetTableId()); + } + immu_mem_list_.clear(); + return Status::OK(); +} + +Status NewMemManager::EraseMemVector(const std::string &table_id) { + {//erase MemVector from rapid-insert cache + std::unique_lock lock(mutex_); + mem_id_map_.erase(table_id); + } + + {//erase MemVector from serialize cache + std::unique_lock lock(serialization_mtx_); + MemList temp_list; + for (auto &mem : immu_mem_list_) { + if (mem->GetTableId() != table_id) { + temp_list.push_back(mem); + } + } + immu_mem_list_.swap(temp_list); + } + + return Status::OK(); +} + +size_t NewMemManager::GetCurrentMutableMem() { + size_t total_mem = 0; + for (auto &kv : mem_id_map_) { + auto memTable = kv.second; + total_mem += memTable->GetCurrentMem(); + } + return total_mem; +} + +size_t NewMemManager::GetCurrentImmutableMem() { + size_t total_mem = 0; + for (auto &mem_table : immu_mem_list_) { + total_mem += mem_table->GetCurrentMem(); + } + return total_mem; +} + +size_t NewMemManager::GetCurrentMem() { + return GetCurrentMutableMem() + GetCurrentImmutableMem(); +} + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/NewMemManager.h b/cpp/src/db/NewMemManager.h new file mode 100644 index 0000000000..5b933c94ca --- /dev/null +++ b/cpp/src/db/NewMemManager.h @@ -0,0 +1,61 @@ +#pragma once + +#include "Meta.h" +#include "MemTable.h" +#include "Status.h" +#include "MemManagerAbstract.h" + +#include +#include +#include +#include +#include + + +namespace zilliz { +namespace milvus { +namespace engine { + +class NewMemManager : public MemManagerAbstract { + public: + using MetaPtr = meta::Meta::Ptr; + using Ptr = std::shared_ptr; + using MemTablePtr = typename MemTable::Ptr; + + NewMemManager(const std::shared_ptr &meta, const Options &options) + : meta_(meta), options_(options) {} + + Status InsertVectors(const std::string &table_id, + size_t n, const float *vectors, IDNumbers &vector_ids) override; + + Status Serialize(std::set &table_ids) override; + + Status EraseMemVector(const std::string &table_id) override; + + size_t GetCurrentMutableMem() override; + + size_t GetCurrentImmutableMem() override; + + size_t GetCurrentMem() override; + + private: + MemTablePtr GetMemByTable(const std::string &table_id); + + Status InsertVectorsNoLock(const std::string &table_id, + size_t n, const float *vectors, IDNumbers &vector_ids); + Status ToImmutable(); + + using MemIdMap = std::map; + using MemList = std::vector; + MemIdMap mem_id_map_; + MemList immu_mem_list_; + MetaPtr meta_; + Options options_; + std::mutex mutex_; + std::mutex serialization_mtx_; +}; // NewMemManager + + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 39d0a15019..515a84d663 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace zilliz { namespace milvus { @@ -43,6 +44,7 @@ private: struct DBMetaOptions { std::string path; + std::vector slave_paths; std::string backend_uri; ArchiveConf archive_conf = ArchiveConf("delete"); }; // DBMetaOptions @@ -61,6 +63,7 @@ struct Options { size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; int mode = MODE::SINGLE; + float maximum_memory = 4 * ONE_GB; }; // Options diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 5f38a9c72a..aa533559bd 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -4,14 +4,58 @@ * Proprietary and confidential. ******************************************************************************/ #include "Utils.h" +#include "utils/CommonUtil.h" +#include "Log.h" +#include #include +#include namespace zilliz { namespace milvus { namespace engine { namespace utils { +namespace { + +static const std::string TABLES_FOLDER = "/tables/"; + +static uint64_t index_file_counter = 0; +static std::mutex index_file_counter_mutex; + +std::string ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) { + std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_; + std::string partition_path = table_path + "/" + std::to_string(table_file.date_); + return partition_path; +} + +std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) { + uint64_t path_count = options.slave_paths.size() + 1; + std::string target_path = options.path; + uint64_t index = 0; + + if(meta::TableFileSchema::INDEX == table_file.file_type_) { + // index file is large file and to be persisted permanently + // we need to distribute index files to each db_path averagely + // round robin according to a file counter + std::lock_guard lock(index_file_counter_mutex); + index = index_file_counter % path_count; + index_file_counter++; + } else { + // for other type files, they could be merged or deleted + // so we round robin according to their file id + index = table_file.id_ % path_count; + } + + if (index > 0) { + target_path = options.slave_paths[index - 1]; + } + + return ConstructParentFolder(target_path, table_file); +} + +} + long GetMicroSecTimeStamp() { auto now = std::chrono::system_clock::now(); auto micros = std::chrono::duration_cast( @@ -20,6 +64,82 @@ long GetMicroSecTimeStamp() { return micros; } +Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id) { + std::string db_path = options.path; + std::string table_path = db_path + TABLES_FOLDER + table_id; + auto status = server::CommonUtil::CreateDirectory(table_path); + if (status != 0) { + ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; + return Status::Error("Failed to create table path"); + } + + for(auto& path : options.slave_paths) { + table_path = path + TABLES_FOLDER + table_id; + status = server::CommonUtil::CreateDirectory(table_path); + if (status != 0) { + ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; + return Status::Error("Failed to create table path"); + } + } + + return Status::OK(); +} + +Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id) { + std::string db_path = options.path; + std::string table_path = db_path + TABLES_FOLDER + table_id; + boost::filesystem::remove_all(table_path); + ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; + + for(auto& path : options.slave_paths) { + table_path = path + TABLES_FOLDER + table_id; + boost::filesystem::remove_all(table_path); + ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; + } + + return Status::OK(); +} + +Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { + std::string parent_path = GetTableFileParentFolder(options, table_file); + + auto status = server::CommonUtil::CreateDirectory(parent_path); + if (status != 0) { + ENGINE_LOG_ERROR << "Create directory " << parent_path << " Error"; + return Status::DBTransactionError("Failed to create partition directory"); + } + + table_file.location_ = parent_path + "/" + table_file.file_id_; + + return Status::OK(); +} + +Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { + std::string parent_path = ConstructParentFolder(options.path, table_file); + std::string file_path = parent_path + "/" + table_file.file_id_; + if(boost::filesystem::exists(file_path)) { + table_file.location_ = file_path; + return Status::OK(); + } else { + for(auto& path : options.slave_paths) { + parent_path = ConstructParentFolder(path, table_file); + file_path = parent_path + "/" + table_file.file_id_; + if(boost::filesystem::exists(file_path)) { + table_file.location_ = file_path; + return Status::OK(); + } + } + } + + return Status::Error("Table file doesn't exist: " + table_file.file_id_); +} + +Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { + utils::GetTableFilePath(options, table_file); + boost::filesystem::remove(table_file.location_); + return Status::OK(); +} + } // namespace utils } // namespace engine } // namespace milvus diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index 481808d630..a5a0f8ff8c 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -5,6 +5,10 @@ ******************************************************************************/ #pragma once +#include "Options.h" +#include "MetaTypes.h" + +#include namespace zilliz { namespace milvus { @@ -13,6 +17,13 @@ namespace utils { long GetMicroSecTimeStamp(); +Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id); +Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id); + +Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); +Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); +Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); + } // namespace utils } // namespace engine } // namespace milvus diff --git a/cpp/src/db/VectorSource.cpp b/cpp/src/db/VectorSource.cpp new file mode 100644 index 0000000000..74c07ae1f6 --- /dev/null +++ b/cpp/src/db/VectorSource.cpp @@ -0,0 +1,67 @@ +#include "VectorSource.h" +#include "ExecutionEngine.h" +#include "EngineFactory.h" +#include "Log.h" +#include "metrics/Metrics.h" + + +namespace zilliz { +namespace milvus { +namespace engine { + + +VectorSource::VectorSource(const size_t &n, + const float *vectors) : + n_(n), + vectors_(vectors), + id_generator_(new SimpleIDGenerator()) { + current_num_vectors_added = 0; +} + +Status VectorSource::Add(const ExecutionEnginePtr &execution_engine, + const meta::TableFileSchema &table_file_schema, + const size_t &num_vectors_to_add, + size_t &num_vectors_added) { + + auto start_time = METRICS_NOW_TIME; + + num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ? + num_vectors_to_add : n_ - current_num_vectors_added; + IDNumbers vector_ids_to_add; + id_generator_->GetNextIDNumbers(num_vectors_added, vector_ids_to_add); + Status status = execution_engine->AddWithIds(num_vectors_added, + vectors_ + current_num_vectors_added * table_file_schema.dimension_, + vector_ids_to_add.data()); + if (status.ok()) { + current_num_vectors_added += num_vectors_added; + vector_ids_.insert(vector_ids_.end(), + std::make_move_iterator(vector_ids_to_add.begin()), + std::make_move_iterator(vector_ids_to_add.end())); + } else { + ENGINE_LOG_ERROR << "VectorSource::Add failed: " + status.ToString(); + } + + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), + static_cast(table_file_schema.dimension_), + total_time); + + return status; +} + +size_t VectorSource::GetNumVectorsAdded() { + return current_num_vectors_added; +} + +bool VectorSource::AllAdded() { + return (current_num_vectors_added == n_); +} + +IDNumbers VectorSource::GetVectorIds() { + return vector_ids_; +} + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/VectorSource.h b/cpp/src/db/VectorSource.h new file mode 100644 index 0000000000..7092805a6d --- /dev/null +++ b/cpp/src/db/VectorSource.h @@ -0,0 +1,46 @@ +#pragma once + +#include "Status.h" +#include "Meta.h" +#include "IDGenerator.h" +#include "ExecutionEngine.h" + + +namespace zilliz { +namespace milvus { +namespace engine { + +class VectorSource { + + public: + + using Ptr = std::shared_ptr; + + VectorSource(const size_t &n, const float *vectors); + + Status Add(const ExecutionEnginePtr &execution_engine, + const meta::TableFileSchema &table_file_schema, + const size_t &num_vectors_to_add, + size_t &num_vectors_added); + + size_t GetNumVectorsAdded(); + + bool AllAdded(); + + IDNumbers GetVectorIds(); + + private: + + const size_t n_; + const float *vectors_; + IDNumbers vector_ids_; + + size_t current_num_vectors_added; + + IDGenerator *id_generator_; + +}; //VectorSource + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 708bcc8708..8036cf986d 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -30,11 +30,20 @@ void CollectDurationMetrics(int index_type, double total_time) { } } +std::string GetMetricType() { + server::ServerConfig &config = server::ServerConfig::GetInstance(); + server::ConfigNode engine_config = config.GetConfig(server::CONFIG_ENGINE); + return engine_config.GetValue(server::CONFIG_METRICTYPE, "L2"); +} + } SearchTask::SearchTask() : IScheduleTask(ScheduleTaskType::kSearch) { - + std::string metric_type = GetMetricType(); + if(metric_type != "L2") { + metric_l2 = false; + } } std::shared_ptr SearchTask::Execute() { @@ -71,7 +80,7 @@ std::shared_ptr SearchTask::Execute() { rc.Record("cluster result"); //step 4: pick up topk result - SearchTask::TopkResult(result_set, inner_k, context->GetResult()); + SearchTask::TopkResult(result_set, inner_k, metric_l2, context->GetResult()); rc.Record("reduce topk"); } catch (std::exception& ex) { @@ -125,7 +134,8 @@ Status SearchTask::ClusterResult(const std::vector &output_ids, Status SearchTask::MergeResult(SearchContext::Id2DistanceMap &distance_src, SearchContext::Id2DistanceMap &distance_target, - uint64_t topk) { + uint64_t topk, + bool ascending) { //Note: the score_src and score_target are already arranged by score in ascending order if(distance_src.empty()) { SERVER_LOG_WARNING << "Empty distance source array"; @@ -161,15 +171,27 @@ Status SearchTask::MergeResult(SearchContext::Id2DistanceMap &distance_src, break; } - //compare score, put smallest score to score_merged one by one + //compare score, + // if ascending = true, put smallest score to score_merged one by one + // else, put largest score to score_merged one by one auto& src_pair = distance_src[src_index]; auto& target_pair = distance_target[target_index]; - if(src_pair.second > target_pair.second) { - distance_merged.push_back(target_pair); - target_index++; + if(ascending){ + if(src_pair.second > target_pair.second) { + distance_merged.push_back(target_pair); + target_index++; + } else { + distance_merged.push_back(src_pair); + src_index++; + } } else { - distance_merged.push_back(src_pair); - src_index++; + if(src_pair.second < target_pair.second) { + distance_merged.push_back(target_pair); + target_index++; + } else { + distance_merged.push_back(src_pair); + src_index++; + } } //score_merged.size() already equal topk @@ -185,6 +207,7 @@ Status SearchTask::MergeResult(SearchContext::Id2DistanceMap &distance_src, Status SearchTask::TopkResult(SearchContext::ResultSet &result_src, uint64_t topk, + bool ascending, SearchContext::ResultSet &result_target) { if (result_target.empty()) { result_target.swap(result_src); @@ -200,7 +223,7 @@ Status SearchTask::TopkResult(SearchContext::ResultSet &result_src, for (size_t i = 0; i < result_src.size(); i++) { SearchContext::Id2DistanceMap &score_src = result_src[i]; SearchContext::Id2DistanceMap &score_target = result_target[i]; - SearchTask::MergeResult(score_src, score_target, topk); + SearchTask::MergeResult(score_src, score_target, topk, ascending); } return Status::OK(); diff --git a/cpp/src/db/scheduler/task/SearchTask.h b/cpp/src/db/scheduler/task/SearchTask.h index e4f0d872b1..7e0aa52af1 100644 --- a/cpp/src/db/scheduler/task/SearchTask.h +++ b/cpp/src/db/scheduler/task/SearchTask.h @@ -27,10 +27,12 @@ public: static Status MergeResult(SearchContext::Id2DistanceMap &distance_src, SearchContext::Id2DistanceMap &distance_target, - uint64_t topk); + uint64_t topk, + bool ascending); static Status TopkResult(SearchContext::ResultSet &result_src, uint64_t topk, + bool ascending, SearchContext::ResultSet &result_target); public: @@ -38,6 +40,7 @@ public: int index_type_ = 0; //for metrics ExecutionEnginePtr index_engine_; std::vector search_contexts_; + bool metric_l2 = true; }; using SearchTaskPtr = std::shared_ptr; diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index b356128898..495e6ae861 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -8,6 +8,7 @@ #include #include +#include #include using namespace ::milvus; @@ -21,7 +22,7 @@ namespace { static constexpr int64_t NQ = 10; static constexpr int64_t TOP_K = 10; static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different - static constexpr int64_t ADD_VECTOR_LOOP = 5; + static constexpr int64_t ADD_VECTOR_LOOP = 10; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; @@ -34,26 +35,17 @@ namespace { BLOCK_SPLITER } - void PrintRecordIdArray(const std::vector& record_ids) { - BLOCK_SPLITER - std::cout << "Returned id array count: " << record_ids.size() << std::endl; -#if 0 - for(auto id : record_ids) { - std::cout << std::to_string(id) << std::endl; - } -#endif - BLOCK_SPLITER - } - - void PrintSearchResult(const std::vector& topk_query_result_array) { + void PrintSearchResult(const std::vector>& search_record_array, + const std::vector& topk_query_result_array) { BLOCK_SPLITER std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl; int32_t index = 0; for(auto& result : topk_query_result_array) { + auto search_id = search_record_array[index].first; index++; - std::cout << "No." << std::to_string(index) << " vector top " - << std::to_string(result.query_result_arrays.size()) + std::cout << "No." << std::to_string(index) << " vector " << std::to_string(search_id) + << " top " << std::to_string(result.query_result_arrays.size()) << " search result:" << std::endl; for(auto& item : result.query_result_arrays) { std::cout << "\t" << std::to_string(item.id) << "\tdistance:" << std::to_string(item.distance); @@ -91,13 +83,13 @@ namespace { std::string GetTableName() { static std::string s_id(CurrentTime()); - return s_id; + return "tbl_" + s_id; } TableSchema BuildTableSchema() { TableSchema tb_schema; tb_schema.table_name = TABLE_NAME; - tb_schema.index_type = IndexType::cpu_idmap; + tb_schema.index_type = IndexType::gpu_ivfflat; tb_schema.dimension = TABLE_DIMENSION; tb_schema.store_raw_vector = true; @@ -126,6 +118,66 @@ namespace { std::cout << "Waiting " << seconds << " seconds ..." << std::endl; sleep(seconds); } + + class TimeRecorder { + public: + TimeRecorder(const std::string& title) + : title_(title) { + start_ = std::chrono::system_clock::now(); + } + + ~TimeRecorder() { + std::chrono::system_clock::time_point end = std::chrono::system_clock::now(); + long span = (std::chrono::duration_cast (end - start_)).count(); + std::cout << title_ << " totally cost: " << span << " ms" << std::endl; + } + + private: + std::string title_; + std::chrono::system_clock::time_point start_; + }; + + void CheckResult(const std::vector>& search_record_array, + const std::vector& topk_query_result_array) { + BLOCK_SPLITER + int64_t index = 0; + for(auto& result : topk_query_result_array) { + auto result_id = result.query_result_arrays[0].id; + auto search_id = search_record_array[index++].first; + if(result_id != search_id) { + std::cout << "The top 1 result is wrong: " << result_id + << " vs. " << search_id << std::endl; + } else { + std::cout << "Check result sucessfully" << std::endl; + } + } + BLOCK_SPLITER + } + + void DoSearch(std::shared_ptr conn, + const std::vector>& search_record_array, + const std::string& phase_name) { + std::vector query_range_array; + Range rg; + rg.start_value = CurrentTmDate(); + rg.end_value = CurrentTmDate(); + query_range_array.emplace_back(rg); + + std::vector record_array; + for(auto& pair : search_record_array) { + record_array.push_back(pair.second); + } + + std::vector topk_query_result_array; + { + TimeRecorder rc(phase_name); + Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array); + std::cout << "SearchVector function call status: " << stat.ToString() << std::endl; + } + + PrintSearchResult(search_record_array, topk_query_result_array); + CheckResult(search_record_array, topk_query_result_array); + } } void @@ -179,30 +231,37 @@ ClientTest::Test(const std::string& address, const std::string& port) { PrintTableSchema(tb_schema); } - for(int i = 0; i < ADD_VECTOR_LOOP; i++){//add vectors - std::vector record_array; - BuildVectors(i*BATCH_ROW_COUNT, (i+1)*BATCH_ROW_COUNT, record_array); - std::vector record_ids; - Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids); - std::cout << "AddVector function call status: " << stat.ToString() << std::endl; - PrintRecordIdArray(record_ids); + std::vector> search_record_array; + {//add vectors + for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors + std::vector record_array; + int64_t begin_index = i * BATCH_ROW_COUNT; + BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array); + std::vector record_ids; + Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids); + std::cout << "AddVector function call status: " << stat.ToString() << std::endl; + std::cout << "Returned id array count: " << record_ids.size() << std::endl; + + if(search_record_array.size() < NQ) { + search_record_array.push_back( + std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); + } + } } - {//search vectors + {//search vectors without index Sleep(2); + DoSearch(conn, search_record_array, "Search without index"); + } - std::vector record_array; - BuildVectors(SEARCH_TARGET, SEARCH_TARGET + NQ, record_array); + {//wait unit build index finish + std::cout << "Wait until build all index done" << std::endl; + Status stat = conn->BuildIndex(TABLE_NAME); + std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl; + } - std::vector query_range_array; - Range rg; - rg.start_value = CurrentTmDate(); - rg.end_value = CurrentTmDate(); - query_range_array.emplace_back(rg); - std::vector topk_query_result_array; - Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array); - std::cout << "SearchVector function call status: " << stat.ToString() << std::endl; - PrintSearchResult(topk_query_result_array); + {//search vectors after build index finish + DoSearch(conn, search_record_array, "Search after build index finish"); } {//delete table diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index cb6d1f753e..a7529d3e00 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -18,6 +18,7 @@ enum class IndexType { invalid = 0, cpu_idmap, gpu_ivfflat, + gpu_ivfsq8, }; /** @@ -180,6 +181,17 @@ public: virtual Status DeleteTable(const std::string &table_name) = 0; + /** + * @brief Build index method + * + * This method is used to build index for whole table + * + * @param table_name, table name is going to be build index. + * + * @return Indicate if build index successfully. + */ + virtual Status BuildIndex(const std::string &table_name) = 0; + /** * @brief Add vector to table * diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index d26007cd0b..81dc8d28e4 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -126,6 +126,22 @@ ClientProxy::DeleteTable(const std::string &table_name) { return Status::OK(); } +Status +ClientProxy::BuildIndex(const std::string &table_name) { + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); + } + + try { + ClientPtr()->interface()->BuildIndex(table_name); + + } catch ( std::exception& ex) { + return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what())); + } + + return Status::OK(); +} + Status ClientProxy::AddVector(const std::string &table_name, const std::vector &record_array, diff --git a/cpp/src/sdk/src/client/ClientProxy.h b/cpp/src/sdk/src/client/ClientProxy.h index a2ede77c40..3e4850ef07 100644 --- a/cpp/src/sdk/src/client/ClientProxy.h +++ b/cpp/src/sdk/src/client/ClientProxy.h @@ -27,6 +27,8 @@ public: virtual Status DeleteTable(const std::string &table_name) override; + virtual Status BuildIndex(const std::string &table_name) override; + virtual Status AddVector(const std::string &table_name, const std::vector &record_array, std::vector &id_array) override; diff --git a/cpp/src/sdk/src/interface/ConnectionImpl.cpp b/cpp/src/sdk/src/interface/ConnectionImpl.cpp index efb5f61b1b..737cd7d83b 100644 --- a/cpp/src/sdk/src/interface/ConnectionImpl.cpp +++ b/cpp/src/sdk/src/interface/ConnectionImpl.cpp @@ -66,6 +66,11 @@ ConnectionImpl::DeleteTable(const std::string &table_name) { return client_proxy_->DeleteTable(table_name); } +Status +ConnectionImpl::BuildIndex(const std::string &table_name) { + return client_proxy_->BuildIndex(table_name); +} + Status ConnectionImpl::AddVector(const std::string &table_name, const std::vector &record_array, diff --git a/cpp/src/sdk/src/interface/ConnectionImpl.h b/cpp/src/sdk/src/interface/ConnectionImpl.h index 0f9cd14e39..60c6cd9b2d 100644 --- a/cpp/src/sdk/src/interface/ConnectionImpl.h +++ b/cpp/src/sdk/src/interface/ConnectionImpl.h @@ -29,6 +29,8 @@ public: virtual Status DeleteTable(const std::string &table_name) override; + virtual Status BuildIndex(const std::string &table_name) override; + virtual Status AddVector(const std::string &table_name, const std::vector &record_array, std::vector &id_array) override; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index fca15cb65a..66a1d83479 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -8,6 +8,7 @@ #include "ServerConfig.h" #include "utils/CommonUtil.h" #include "utils/Log.h" +#include "utils/StringHelpFunctions.h" namespace zilliz { namespace milvus { @@ -19,10 +20,22 @@ DBWrapper::DBWrapper() { opt.meta.backend_uri = config.GetValue(CONFIG_DB_URL); std::string db_path = config.GetValue(CONFIG_DB_PATH); opt.meta.path = db_path + "/db"; + + std::string db_slave_path = config.GetValue(CONFIG_DB_SLAVE_PATH); + StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths); + int64_t index_size = config.GetInt64Value(CONFIG_DB_INDEX_TRIGGER_SIZE); if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } + float maximum_memory = config.GetFloatValue(CONFIG_MAXMIMUM_MEMORY); + if (maximum_memory > 1.0) { + opt.maximum_memory = maximum_memory * engine::ONE_GB; + } + else { + std::cout << "ERROR: maximum_memory should be at least 1 GB" << std::endl; + kill(0, SIGUSR1); + } ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); @@ -59,6 +72,14 @@ DBWrapper::DBWrapper() { kill(0, SIGUSR1); } + for(auto& path : opt.meta.slave_paths) { + err = CommonUtil::CreateDirectory(path); + if(err != SERVER_SUCCESS) { + std::cout << "ERROR! Failed to create database slave path: " << path << std::endl; + kill(0, SIGUSR1); + } + } + std::string msg = opt.meta.path; try { zilliz::milvus::engine::DB::Open(opt, &db_); diff --git a/cpp/src/server/MilvusServer.cpp b/cpp/src/server/MilvusServer.cpp index 452ee3af88..92e06d0ccd 100644 --- a/cpp/src/server/MilvusServer.cpp +++ b/cpp/src/server/MilvusServer.cpp @@ -8,9 +8,11 @@ #include "ServerConfig.h" #include "ThreadPoolServer.h" #include "DBWrapper.h" +#include "utils/Log.h" #include "milvus_types.h" #include "milvus_constants.h" +#include "faiss/utils.h" #include #include @@ -25,6 +27,8 @@ #include #include +//extern int distance_compute_blas_threshold; + namespace zilliz { namespace milvus { namespace server { @@ -46,11 +50,12 @@ MilvusServer::StartService() { ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_SERVER); - + ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, "127.0.0.1"); int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, 19530); std::string protocol = server_config.GetValue(CONFIG_SERVER_PROTOCOL, "binary"); - + faiss::distance_compute_blas_threshold = engine_config.GetInt32Value(CONFIG_DCBT,20); +// std::cout<<"distance_compute_blas_threshold = "<< faiss::distance_compute_blas_threshold << std::endl; try { DBWrapper::DB();//initialize db diff --git a/cpp/src/server/RequestHandler.cpp b/cpp/src/server/RequestHandler.cpp index a4dc182c35..5ff6b1784a 100644 --- a/cpp/src/server/RequestHandler.cpp +++ b/cpp/src/server/RequestHandler.cpp @@ -39,6 +39,12 @@ RequestHandler::DeleteTable(const std::string &table_name) { RequestScheduler::ExecTask(task_ptr); } +void +RequestHandler::BuildIndex(const std::string &table_name) { + BaseTaskPtr task_ptr = BuildIndexTask::Create(table_name); + RequestScheduler::ExecTask(task_ptr); +} + void RequestHandler::AddVector(std::vector &_return, const std::string &table_name, diff --git a/cpp/src/server/RequestHandler.h b/cpp/src/server/RequestHandler.h index e736b7593f..aeda4e5ed6 100644 --- a/cpp/src/server/RequestHandler.h +++ b/cpp/src/server/RequestHandler.h @@ -54,6 +54,18 @@ public: */ void DeleteTable(const std::string& table_name); + /** + * @brief build index by table method + * + * This method is used to build index by table in sync. + * + * @param table_name, table name is going to be built index. + * + * + * @param table_name + */ + void BuildIndex(const std::string &table_name); + /** * @brief Add vector array to table * diff --git a/cpp/src/server/RequestScheduler.cpp b/cpp/src/server/RequestScheduler.cpp index 36df155b62..73c09c8bd4 100644 --- a/cpp/src/server/RequestScheduler.cpp +++ b/cpp/src/server/RequestScheduler.cpp @@ -12,7 +12,7 @@ namespace zilliz { namespace milvus { namespace server { - + using namespace ::milvus; namespace { @@ -43,6 +43,7 @@ namespace { {SERVER_ILLEGAL_SEARCH_RESULT, thrift::ErrorCode::ILLEGAL_SEARCH_RESULT}, {SERVER_CACHE_ERROR, thrift::ErrorCode::CACHE_FAILED}, {DB_META_TRANSACTION_FAILED, thrift::ErrorCode::META_FAILED}, + {SERVER_BUILD_INDEX_ERROR, thrift::ErrorCode::BUILD_INDEX_ERROR}, }; return code_map; diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 1b91883af5..d4051eba66 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -8,6 +8,7 @@ #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" #include "DBWrapper.h" #include "version.h" @@ -30,6 +31,7 @@ namespace { {0, engine::EngineType::INVALID}, {1, engine::EngineType::FAISS_IDMAP}, {2, engine::EngineType::FAISS_IVFFLAT}, + {3, engine::EngineType::FAISS_IVFSQ8}, }; if(map_type.find(type) == map_type.end()) { @@ -44,6 +46,7 @@ namespace { {engine::EngineType::INVALID, 0}, {engine::EngineType::FAISS_IDMAP, 1}, {engine::EngineType::FAISS_IVFFLAT, 2}, + {engine::EngineType::FAISS_IVFSQ8, 3}, }; if(map_type.find(type) == map_type.end()) { @@ -108,7 +111,13 @@ namespace { } long days = (tt_end > tt_start) ? (tt_end - tt_start)/DAY_SECONDS : (tt_start - tt_end)/DAY_SECONDS; - for(long i = 0; i <= days; i++) { + if(days == 0) { + error_code = SERVER_INVALID_TIME_RANGE; + error_msg = "Invalid time range: " + range.start_value + " to " + range.end_value; + return ; + } + + for(long i = 0; i < days; i++) { time_t tt_day = tt_start + DAY_SECONDS*i; tm tm_day; CommonUtil::ConvertTime(tt_day, tm_day); @@ -133,19 +142,23 @@ BaseTaskPtr CreateTableTask::Create(const thrift::TableSchema& schema) { ServerError CreateTableTask::OnExecute() { TimeRecorder rc("CreateTableTask"); - + try { //step 1: check arguments - if(schema_.table_name.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); - } - if(schema_.dimension <= 0) { - return SetError(SERVER_INVALID_TABLE_DIMENSION, "Invalid table dimension: " + std::to_string(schema_.dimension)); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(schema_.table_name); + if(res != SERVER_SUCCESS) { + return res; } - engine::EngineType engine_type = EngineType(schema_.index_type); - if(engine_type == engine::EngineType::INVALID) { - return SetError(SERVER_INVALID_INDEX_TYPE, "Invalid index type: " + std::to_string(schema_.index_type)); + res = ValidateTableDimension(schema_.dimension); + if(res != SERVER_SUCCESS) { + return res; + } + + res = ValidateTableIndexType(schema_.index_type); + if(res != SERVER_SUCCESS) { + return res; } //step 2: construct table schema @@ -187,8 +200,10 @@ ServerError DescribeTableTask::OnExecute() { try { //step 1: check arguments - if(table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return res; } //step 2: get table info @@ -213,6 +228,39 @@ ServerError DescribeTableTask::OnExecute() { return SERVER_SUCCESS; } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +BuildIndexTask::BuildIndexTask(const std::string& table_name) + : BaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name) { +} + +BaseTaskPtr BuildIndexTask::Create(const std::string& table_name) { + return std::shared_ptr(new BuildIndexTask(table_name)); +} + +ServerError BuildIndexTask::OnExecute() { + try { + TimeRecorder rc("BuildIndexTask"); + + //step 1: check arguments + if(table_name_.empty()) { + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + } + + //step 2: check table existence + engine::Status stat = DBWrapper::DB()->BuildIndex(table_name_); + if(!stat.ok()) { + return SetError(SERVER_BUILD_INDEX_ERROR, "Engine failed: " + stat.ToString()); + } + + rc.Elapse("totally cost"); + } catch (std::exception& ex) { + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return SERVER_SUCCESS; +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// HasTableTask::HasTableTask(const std::string& table_name, bool& has_table) : BaseTask(DDL_DML_TASK_GROUP), @@ -230,10 +278,11 @@ ServerError HasTableTask::OnExecute() { TimeRecorder rc("HasTableTask"); //step 1: check arguments - if(table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return res; } - //step 2: check table existence engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_); if(!stat.ok()) { @@ -264,8 +313,10 @@ ServerError DeleteTableTask::OnExecute() { TimeRecorder rc("DeleteTableTask"); //step 1: check arguments - if (table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return res; } //step 2: check table existence @@ -290,7 +341,7 @@ ServerError DeleteTableTask::OnExecute() { } rc.Record("deleta table"); - rc.Elapse("totally cost"); + rc.Elapse("total cost"); } catch (std::exception& ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -346,8 +397,10 @@ ServerError AddVectorTask::OnExecute() { TimeRecorder rc("AddVectorTask"); //step 1: check arguments - if (table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return res; } if(record_array_.empty()) { @@ -394,7 +447,7 @@ ServerError AddVectorTask::OnExecute() { } rc.Record("do insert"); - rc.Elapse("totally cost"); + rc.Elapse("total cost"); } catch (std::exception& ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); @@ -435,8 +488,10 @@ ServerError SearchVectorTask::OnExecute() { TimeRecorder rc("SearchVectorTask"); //step 1: check arguments - if (table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return res; } if(top_k_ <= 0) { @@ -522,7 +577,7 @@ ServerError SearchVectorTask::OnExecute() { result_array_.emplace_back(thrift_topk_result); } rc.Record("construct result"); - rc.Elapse("totally cost"); + rc.Elapse("total cost"); } catch (std::exception& ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); @@ -548,8 +603,10 @@ ServerError GetTableRowCountTask::OnExecute() { TimeRecorder rc("GetTableRowCountTask"); //step 1: check arguments - if (table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return res; } //step 2: get row count @@ -561,7 +618,7 @@ ServerError GetTableRowCountTask::OnExecute() { row_count_ = (int64_t) row_count; - rc.Elapse("totally cost"); + rc.Elapse("total cost"); } catch (std::exception& ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); diff --git a/cpp/src/server/RequestTask.h b/cpp/src/server/RequestTask.h index 4bf9f964e0..37c04272fa 100644 --- a/cpp/src/server/RequestTask.h +++ b/cpp/src/server/RequestTask.h @@ -75,6 +75,21 @@ protected: ServerError OnExecute() override; +private: + std::string table_name_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class BuildIndexTask : public BaseTask { +public: + static BaseTaskPtr Create(const std::string& table_name); + +protected: + BuildIndexTask(const std::string& table_name); + + ServerError OnExecute() override; + + private: std::string table_name_; }; @@ -174,4 +189,4 @@ private: } } -} \ No newline at end of file +} diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index 0ec04eed8c..d3fb77a8cb 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -23,9 +23,11 @@ static const std::string CONFIG_CLUSTER_MODE = "mode"; static const std::string CONFIG_DB = "db_config"; static const std::string CONFIG_DB_URL = "db_backend_url"; static const std::string CONFIG_DB_PATH = "db_path"; +static const std::string CONFIG_DB_SLAVE_PATH = "db_slave_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; +static const std::string CONFIG_MAXMIMUM_MEMORY = "maximum_memory"; static const std::string CONFIG_LOG = "log_config"; @@ -44,6 +46,9 @@ static const std::string CONFIG_METRIC_PROMETHEUS_PORT = "port"; static const std::string CONFIG_ENGINE = "engine_config"; static const std::string CONFIG_NPROBE = "nprobe"; +static const std::string CONFIG_NLIST = "nlist"; +static const std::string CONFIG_DCBT = "use_blas_threshold"; +static const std::string CONFIG_METRICTYPE = "metric_type"; class ServerConfig { public: diff --git a/cpp/src/thrift/gen-cpp/MilvusService.cpp b/cpp/src/thrift/gen-cpp/MilvusService.cpp index 7a591276f6..fb66b0ecbe 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService.cpp +++ b/cpp/src/thrift/gen-cpp/MilvusService.cpp @@ -590,6 +590,193 @@ uint32_t MilvusService_DeleteTable_presult::read(::apache::thrift::protocol::TPr } +MilvusService_BuildIndex_args::~MilvusService_BuildIndex_args() throw() { +} + + +uint32_t MilvusService_BuildIndex_args::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->table_name); + this->__isset.table_name = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t MilvusService_BuildIndex_args::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("MilvusService_BuildIndex_args"); + + xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->table_name); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_BuildIndex_pargs::~MilvusService_BuildIndex_pargs() throw() { +} + + +uint32_t MilvusService_BuildIndex_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("MilvusService_BuildIndex_pargs"); + + xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString((*(this->table_name))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_BuildIndex_result::~MilvusService_BuildIndex_result() throw() { +} + + +uint32_t MilvusService_BuildIndex_result::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->e.read(iprot); + this->__isset.e = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t MilvusService_BuildIndex_result::write(::apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("MilvusService_BuildIndex_result"); + + if (this->__isset.e) { + xfer += oprot->writeFieldBegin("e", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += this->e.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_BuildIndex_presult::~MilvusService_BuildIndex_presult() throw() { +} + + +uint32_t MilvusService_BuildIndex_presult::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->e.read(iprot); + this->__isset.e = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + + MilvusService_AddVector_args::~MilvusService_AddVector_args() throw() { } @@ -2614,6 +2801,62 @@ void MilvusServiceClient::recv_DeleteTable() return; } +void MilvusServiceClient::BuildIndex(const std::string& table_name) +{ + send_BuildIndex(table_name); + recv_BuildIndex(); +} + +void MilvusServiceClient::send_BuildIndex(const std::string& table_name) +{ + int32_t cseqid = 0; + oprot_->writeMessageBegin("BuildIndex", ::apache::thrift::protocol::T_CALL, cseqid); + + MilvusService_BuildIndex_pargs args; + args.table_name = &table_name; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); +} + +void MilvusServiceClient::recv_BuildIndex() +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("BuildIndex") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + MilvusService_BuildIndex_presult result; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.e) { + throw result.e; + } + return; +} + void MilvusServiceClient::AddVector(std::vector & _return, const std::string& table_name, const std::vector & record_array) { send_AddVector(table_name, record_array); @@ -3236,6 +3479,62 @@ void MilvusServiceProcessor::process_DeleteTable(int32_t seqid, ::apache::thrift } } +void MilvusServiceProcessor::process_BuildIndex(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +{ + void* ctx = NULL; + if (this->eventHandler_.get() != NULL) { + ctx = this->eventHandler_->getContext("MilvusService.BuildIndex", callContext); + } + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "MilvusService.BuildIndex"); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preRead(ctx, "MilvusService.BuildIndex"); + } + + MilvusService_BuildIndex_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postRead(ctx, "MilvusService.BuildIndex", bytes); + } + + MilvusService_BuildIndex_result result; + try { + iface_->BuildIndex(args.table_name); + } catch (Exception &e) { + result.e = e; + result.__isset.e = true; + } catch (const std::exception& e) { + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->handlerError(ctx, "MilvusService.BuildIndex"); + } + + ::apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("BuildIndex", ::apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preWrite(ctx, "MilvusService.BuildIndex"); + } + + oprot->writeMessageBegin("BuildIndex", ::apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postWrite(ctx, "MilvusService.BuildIndex", bytes); + } +} + void MilvusServiceProcessor::process_AddVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) { void* ctx = NULL; @@ -3894,6 +4193,88 @@ void MilvusServiceConcurrentClient::recv_DeleteTable(const int32_t seqid) } // end while(true) } +void MilvusServiceConcurrentClient::BuildIndex(const std::string& table_name) +{ + int32_t seqid = send_BuildIndex(table_name); + recv_BuildIndex(seqid); +} + +int32_t MilvusServiceConcurrentClient::send_BuildIndex(const std::string& table_name) +{ + int32_t cseqid = this->sync_.generateSeqId(); + ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); + oprot_->writeMessageBegin("BuildIndex", ::apache::thrift::protocol::T_CALL, cseqid); + + MilvusService_BuildIndex_pargs args; + args.table_name = &table_name; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + + sentry.commit(); + return cseqid; +} + +void MilvusServiceConcurrentClient::recv_BuildIndex(const int32_t seqid) +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + // the read mutex gets dropped and reacquired as part of waitForWork() + // The destructor of this sentry wakes up other clients + ::apache::thrift::async::TConcurrentRecvSentry sentry(&this->sync_, seqid); + + while(true) { + if(!this->sync_.getPending(fname, mtype, rseqid)) { + iprot_->readMessageBegin(fname, mtype, rseqid); + } + if(seqid == rseqid) { + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + sentry.commit(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("BuildIndex") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + // in a bad state, don't commit + using ::apache::thrift::protocol::TProtocolException; + throw TProtocolException(TProtocolException::INVALID_DATA); + } + MilvusService_BuildIndex_presult result; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.e) { + sentry.commit(); + throw result.e; + } + sentry.commit(); + return; + } + // seqid != rseqid + this->sync_.updatePending(fname, mtype, rseqid); + + // this will temporarily unlock the readMutex, and let other clients get work done + this->sync_.waitForWork(seqid); + } // end while(true) +} + void MilvusServiceConcurrentClient::AddVector(std::vector & _return, const std::string& table_name, const std::vector & record_array) { int32_t seqid = send_AddVector(table_name, record_array); diff --git a/cpp/src/thrift/gen-cpp/MilvusService.h b/cpp/src/thrift/gen-cpp/MilvusService.h index 3868681463..5b4204e6b8 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService.h +++ b/cpp/src/thrift/gen-cpp/MilvusService.h @@ -58,6 +58,18 @@ class MilvusServiceIf { */ virtual void DeleteTable(const std::string& table_name) = 0; + /** + * @brief Build index by table method + * + * This method is used to build index by table in sync mode. + * + * @param table_name, table is going to be built index. + * + * + * @param table_name + */ + virtual void BuildIndex(const std::string& table_name) = 0; + /** * @brief Add vector array to table * @@ -197,6 +209,9 @@ class MilvusServiceNull : virtual public MilvusServiceIf { void DeleteTable(const std::string& /* table_name */) { return; } + void BuildIndex(const std::string& /* table_name */) { + return; + } void AddVector(std::vector & /* _return */, const std::string& /* table_name */, const std::vector & /* record_array */) { return; } @@ -541,6 +556,110 @@ class MilvusService_DeleteTable_presult { }; +typedef struct _MilvusService_BuildIndex_args__isset { + _MilvusService_BuildIndex_args__isset() : table_name(false) {} + bool table_name :1; +} _MilvusService_BuildIndex_args__isset; + +class MilvusService_BuildIndex_args { + public: + + MilvusService_BuildIndex_args(const MilvusService_BuildIndex_args&); + MilvusService_BuildIndex_args& operator=(const MilvusService_BuildIndex_args&); + MilvusService_BuildIndex_args() : table_name() { + } + + virtual ~MilvusService_BuildIndex_args() throw(); + std::string table_name; + + _MilvusService_BuildIndex_args__isset __isset; + + void __set_table_name(const std::string& val); + + bool operator == (const MilvusService_BuildIndex_args & rhs) const + { + if (!(table_name == rhs.table_name)) + return false; + return true; + } + bool operator != (const MilvusService_BuildIndex_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MilvusService_BuildIndex_args & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + + +class MilvusService_BuildIndex_pargs { + public: + + + virtual ~MilvusService_BuildIndex_pargs() throw(); + const std::string* table_name; + + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _MilvusService_BuildIndex_result__isset { + _MilvusService_BuildIndex_result__isset() : e(false) {} + bool e :1; +} _MilvusService_BuildIndex_result__isset; + +class MilvusService_BuildIndex_result { + public: + + MilvusService_BuildIndex_result(const MilvusService_BuildIndex_result&); + MilvusService_BuildIndex_result& operator=(const MilvusService_BuildIndex_result&); + MilvusService_BuildIndex_result() { + } + + virtual ~MilvusService_BuildIndex_result() throw(); + Exception e; + + _MilvusService_BuildIndex_result__isset __isset; + + void __set_e(const Exception& val); + + bool operator == (const MilvusService_BuildIndex_result & rhs) const + { + if (!(e == rhs.e)) + return false; + return true; + } + bool operator != (const MilvusService_BuildIndex_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MilvusService_BuildIndex_result & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _MilvusService_BuildIndex_presult__isset { + _MilvusService_BuildIndex_presult__isset() : e(false) {} + bool e :1; +} _MilvusService_BuildIndex_presult__isset; + +class MilvusService_BuildIndex_presult { + public: + + + virtual ~MilvusService_BuildIndex_presult() throw(); + Exception e; + + _MilvusService_BuildIndex_presult__isset __isset; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + +}; + typedef struct _MilvusService_AddVector_args__isset { _MilvusService_AddVector_args__isset() : table_name(false), record_array(false) {} bool table_name :1; @@ -1403,6 +1522,9 @@ class MilvusServiceClient : virtual public MilvusServiceIf { void DeleteTable(const std::string& table_name); void send_DeleteTable(const std::string& table_name); void recv_DeleteTable(); + void BuildIndex(const std::string& table_name); + void send_BuildIndex(const std::string& table_name); + void recv_BuildIndex(); void AddVector(std::vector & _return, const std::string& table_name, const std::vector & record_array); void send_AddVector(const std::string& table_name, const std::vector & record_array); void recv_AddVector(std::vector & _return); @@ -1442,6 +1564,7 @@ class MilvusServiceProcessor : public ::apache::thrift::TDispatchProcessor { void process_CreateTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_HasTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_DeleteTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); + void process_BuildIndex(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_AddVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_SearchVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_SearchVectorInFiles(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); @@ -1455,6 +1578,7 @@ class MilvusServiceProcessor : public ::apache::thrift::TDispatchProcessor { processMap_["CreateTable"] = &MilvusServiceProcessor::process_CreateTable; processMap_["HasTable"] = &MilvusServiceProcessor::process_HasTable; processMap_["DeleteTable"] = &MilvusServiceProcessor::process_DeleteTable; + processMap_["BuildIndex"] = &MilvusServiceProcessor::process_BuildIndex; processMap_["AddVector"] = &MilvusServiceProcessor::process_AddVector; processMap_["SearchVector"] = &MilvusServiceProcessor::process_SearchVector; processMap_["SearchVectorInFiles"] = &MilvusServiceProcessor::process_SearchVectorInFiles; @@ -1517,6 +1641,15 @@ class MilvusServiceMultiface : virtual public MilvusServiceIf { ifaces_[i]->DeleteTable(table_name); } + void BuildIndex(const std::string& table_name) { + size_t sz = ifaces_.size(); + size_t i = 0; + for (; i < (sz - 1); ++i) { + ifaces_[i]->BuildIndex(table_name); + } + ifaces_[i]->BuildIndex(table_name); + } + void AddVector(std::vector & _return, const std::string& table_name, const std::vector & record_array) { size_t sz = ifaces_.size(); size_t i = 0; @@ -1625,6 +1758,9 @@ class MilvusServiceConcurrentClient : virtual public MilvusServiceIf { void DeleteTable(const std::string& table_name); int32_t send_DeleteTable(const std::string& table_name); void recv_DeleteTable(const int32_t seqid); + void BuildIndex(const std::string& table_name); + int32_t send_BuildIndex(const std::string& table_name); + void recv_BuildIndex(const int32_t seqid); void AddVector(std::vector & _return, const std::string& table_name, const std::vector & record_array); int32_t send_AddVector(const std::string& table_name, const std::vector & record_array); void recv_AddVector(std::vector & _return, const int32_t seqid); diff --git a/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp b/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp index ecb22c0b62..f22cfc2894 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp +++ b/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp @@ -65,6 +65,21 @@ class MilvusServiceHandler : virtual public MilvusServiceIf { printf("DeleteTable\n"); } + /** + * @brief Build index by table method + * + * This method is used to build index by table in sync mode. + * + * @param table_name, table is going to be built index. + * + * + * @param table_name + */ + void BuildIndex(const std::string& table_name) { + // Your implementation goes here + printf("BuildIndex\n"); + } + /** * @brief Add vector array to table * diff --git a/cpp/src/thrift/gen-cpp/milvus_types.cpp b/cpp/src/thrift/gen-cpp/milvus_types.cpp index 09aedec0bf..06923c3a1a 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.cpp +++ b/cpp/src/thrift/gen-cpp/milvus_types.cpp @@ -34,7 +34,8 @@ int _kErrorCodeValues[] = { ErrorCode::CANNOT_CREATE_FOLDER, ErrorCode::CANNOT_CREATE_FILE, ErrorCode::CANNOT_DELETE_FOLDER, - ErrorCode::CANNOT_DELETE_FILE + ErrorCode::CANNOT_DELETE_FILE, + ErrorCode::BUILD_INDEX_ERROR }; const char* _kErrorCodeNames[] = { "SUCCESS", @@ -57,9 +58,10 @@ const char* _kErrorCodeNames[] = { "CANNOT_CREATE_FOLDER", "CANNOT_CREATE_FILE", "CANNOT_DELETE_FOLDER", - "CANNOT_DELETE_FILE" + "CANNOT_DELETE_FILE", + "BUILD_INDEX_ERROR" }; -const std::map _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(21, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); +const std::map _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(22, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); std::ostream& operator<<(std::ostream& out, const ErrorCode::type& val) { std::map::const_iterator it = _ErrorCode_VALUES_TO_NAMES.find(val); diff --git a/cpp/src/thrift/gen-cpp/milvus_types.h b/cpp/src/thrift/gen-cpp/milvus_types.h index 34c5e084ce..63e313d197 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.h +++ b/cpp/src/thrift/gen-cpp/milvus_types.h @@ -42,7 +42,8 @@ struct ErrorCode { CANNOT_CREATE_FOLDER = 17, CANNOT_CREATE_FILE = 18, CANNOT_DELETE_FOLDER = 19, - CANNOT_DELETE_FILE = 20 + CANNOT_DELETE_FILE = 20, + BUILD_INDEX_ERROR = 21 }; }; diff --git a/cpp/src/thrift/milvus.thrift b/cpp/src/thrift/milvus.thrift index 813026fdd8..001b27b04b 100644 --- a/cpp/src/thrift/milvus.thrift +++ b/cpp/src/thrift/milvus.thrift @@ -35,6 +35,7 @@ enum ErrorCode { CANNOT_CREATE_FILE, CANNOT_DELETE_FOLDER, CANNOT_DELETE_FILE, + BUILD_INDEX_ERROR, } exception Exception { @@ -115,6 +116,16 @@ service MilvusService { */ void DeleteTable(2: string table_name) throws(1: Exception e); + /** + * @brief Build index by table method + * + * This method is used to build index by table in sync mode. + * + * @param table_name, table is going to be built index. + * + */ + void BuildIndex(2: string table_name) throws(1: Exception e); + /** * @brief Add vector array to table @@ -207,4 +218,4 @@ service MilvusService { * @return Server status. */ string Ping(2: string cmd) throws(1: Exception e); -} \ No newline at end of file +} diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index 19157af9f2..a376ee536f 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -32,27 +32,26 @@ namespace server { namespace fs = boost::filesystem; -bool CommonUtil::GetSystemMemInfo(unsigned long &totalMem, unsigned long &freeMem) { +bool CommonUtil::GetSystemMemInfo(unsigned long &total_mem, unsigned long &free_mem) { struct sysinfo info; int ret = sysinfo(&info); - totalMem = info.totalram; - freeMem = info.freeram; + total_mem = info.totalram; + free_mem = info.freeram; return ret == 0;//succeed 0, failed -1 } -bool CommonUtil::GetSystemAvailableThreads(unsigned int &threadCnt) { +bool CommonUtil::GetSystemAvailableThreads(unsigned int &thread_count) { //threadCnt = std::thread::hardware_concurrency(); - threadCnt = sysconf(_SC_NPROCESSORS_CONF); - threadCnt *= THREAD_MULTIPLY_CPU; - if (threadCnt == 0) - threadCnt = 8; + thread_count = sysconf(_SC_NPROCESSORS_CONF); + thread_count *= THREAD_MULTIPLY_CPU; + if (thread_count == 0) + thread_count = 8; return true; } -bool CommonUtil::IsDirectoryExist(const std::string &path) -{ +bool CommonUtil::IsDirectoryExist(const std::string &path) { DIR *dp = nullptr; if ((dp = opendir(path.c_str())) == nullptr) { return false; @@ -63,9 +62,13 @@ bool CommonUtil::IsDirectoryExist(const std::string &path) } ServerError CommonUtil::CreateDirectory(const std::string &path) { - struct stat directoryStat; - int statOK = stat(path.c_str(), &directoryStat); - if (statOK == 0) { + if(path.empty()) { + return SERVER_SUCCESS; + } + + struct stat directory_stat; + int status = stat(path.c_str(), &directory_stat); + if (status == 0) { return SERVER_SUCCESS;//already exist } @@ -76,8 +79,8 @@ ServerError CommonUtil::CreateDirectory(const std::string &path) { return err; } - statOK = stat(path.c_str(), &directoryStat); - if (statOK == 0) { + status = stat(path.c_str(), &directory_stat); + if (status == 0) { return SERVER_SUCCESS;//already exist } @@ -91,37 +94,41 @@ ServerError CommonUtil::CreateDirectory(const std::string &path) { namespace { void RemoveDirectory(const std::string &path) { - DIR *pDir = NULL; + DIR *dir = nullptr; struct dirent *dmsg; - char szFileName[256]; - char szFolderName[256]; + char file_name[256]; + char folder_name[256]; - strcpy(szFolderName, path.c_str()); - strcat(szFolderName, "/%s"); - if ((pDir = opendir(path.c_str())) != NULL) { - while ((dmsg = readdir(pDir)) != NULL) { + strcpy(folder_name, path.c_str()); + strcat(folder_name, "/%s"); + if ((dir = opendir(path.c_str())) != nullptr) { + while ((dmsg = readdir(dir)) != nullptr) { if (strcmp(dmsg->d_name, ".") != 0 && strcmp(dmsg->d_name, "..") != 0) { - sprintf(szFileName, szFolderName, dmsg->d_name); - std::string tmp = szFileName; + sprintf(file_name, folder_name, dmsg->d_name); + std::string tmp = file_name; if (tmp.find(".") == std::string::npos) { - RemoveDirectory(szFileName); + RemoveDirectory(file_name); } - remove(szFileName); + remove(file_name); } } } - if (pDir != NULL) { - closedir(pDir); + if (dir != nullptr) { + closedir(dir); } remove(path.c_str()); } } ServerError CommonUtil::DeleteDirectory(const std::string &path) { - struct stat directoryStat; - int statOK = stat(path.c_str(), &directoryStat); + if(path.empty()) { + return SERVER_SUCCESS; + } + + struct stat directory_stat; + int statOK = stat(path.c_str(), &directory_stat); if (statOK != 0) return SERVER_SUCCESS; @@ -133,6 +140,15 @@ bool CommonUtil::IsFileExist(const std::string &path) { return (access(path.c_str(), F_OK) == 0); } +uint64_t CommonUtil::GetFileSize(const std::string &path) { + struct stat file_info; + if (stat(path.c_str(), &file_info) < 0) { + return 0; + } else { + return (uint64_t)file_info.st_size; + } +} + std::string CommonUtil::GetExePath() { const size_t buf_len = 1024; char buf[buf_len]; diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index f1db3c059d..095b237584 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -16,10 +16,11 @@ namespace server { class CommonUtil { public: - static bool GetSystemMemInfo(unsigned long &totalMem, unsigned long &freeMem); - static bool GetSystemAvailableThreads(unsigned int &threadCnt); + static bool GetSystemMemInfo(unsigned long &total_mem, unsigned long &free_mem); + static bool GetSystemAvailableThreads(unsigned int &thread_count); static bool IsFileExist(const std::string &path); + static uint64_t GetFileSize(const std::string &path); static bool IsDirectoryExist(const std::string &path); static ServerError CreateDirectory(const std::string &path); static ServerError DeleteDirectory(const std::string &path); diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index 8c4da70339..3b5433e76f 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -35,6 +35,7 @@ constexpr ServerError SERVER_CANNOT_CREATE_FOLDER = ToGlobalServerErrorCode(8); constexpr ServerError SERVER_CANNOT_CREATE_FILE = ToGlobalServerErrorCode(9); constexpr ServerError SERVER_CANNOT_DELETE_FOLDER = ToGlobalServerErrorCode(10); constexpr ServerError SERVER_CANNOT_DELETE_FILE = ToGlobalServerErrorCode(11); +constexpr ServerError SERVER_BUILD_INDEX_ERROR = ToGlobalServerErrorCode(12); constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(100); constexpr ServerError SERVER_INVALID_TABLE_NAME = ToGlobalServerErrorCode(101); @@ -77,4 +78,3 @@ private: } // namespace server } // namespace milvus } // namespace zilliz - diff --git a/cpp/src/utils/StringHelpFunctions.cpp b/cpp/src/utils/StringHelpFunctions.cpp index 36ba017beb..03c689d82c 100644 --- a/cpp/src/utils/StringHelpFunctions.cpp +++ b/cpp/src/utils/StringHelpFunctions.cpp @@ -27,6 +27,10 @@ void StringHelpFunctions::TrimStringQuote(std::string &string, const std::string ServerError StringHelpFunctions::SplitStringByDelimeter(const std::string &str, const std::string &delimeter, std::vector &result) { + if(str.empty()) { + return SERVER_SUCCESS; + } + size_t last = 0; size_t index = str.find_first_of(delimeter, last); while (index != std::string::npos) { diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp new file mode 100644 index 0000000000..65cd81e670 --- /dev/null +++ b/cpp/src/utils/ValidationUtil.cpp @@ -0,0 +1,74 @@ +#include +#include "ValidationUtil.h" +#include "Log.h" + + +namespace zilliz { +namespace milvus { +namespace server { + +constexpr size_t table_name_size_limit = 255; +constexpr int64_t table_dimension_limit = 16384; + +ServerError +ValidateTableName(const std::string &table_name) { + + // Table name shouldn't be empty. + if (table_name.empty()) { + SERVER_LOG_ERROR << "Empty table name"; + return SERVER_INVALID_TABLE_NAME; + } + + // Table name size shouldn't exceed 16384. + if (table_name.size() > table_name_size_limit) { + SERVER_LOG_ERROR << "Table name size exceed the limitation"; + return SERVER_INVALID_TABLE_NAME; + } + + // Table name first character should be underscore or character. + char first_char = table_name[0]; + if (first_char != '_' && std::isalpha(first_char) == 0) { + SERVER_LOG_ERROR << "Table name first character isn't underscore or character: " << first_char; + return SERVER_INVALID_TABLE_NAME; + } + + int64_t table_name_size = table_name.size(); + for (int64_t i = 1; i < table_name_size; ++i) { + char name_char = table_name[i]; + if (name_char != '_' && std::isalnum(name_char) == 0) { + SERVER_LOG_ERROR << "Table name character isn't underscore or alphanumber: " << name_char; + return SERVER_INVALID_TABLE_NAME; + } + } + + return SERVER_SUCCESS; +} + +ServerError +ValidateTableDimension(int64_t dimension) { + if (dimension <= 0 || dimension > table_dimension_limit) { + SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit; + return SERVER_INVALID_VECTOR_DIMENSION; + } else { + return SERVER_SUCCESS; + } +} + +ServerError +ValidateTableIndexType(int32_t index_type) { + auto engine_type = engine::EngineType(index_type); + switch (engine_type) { + case engine::EngineType::FAISS_IDMAP: + case engine::EngineType::FAISS_IVFFLAT: { + SERVER_LOG_DEBUG << "Index type: " << index_type; + return SERVER_SUCCESS; + } + default: { + return SERVER_INVALID_INDEX_TYPE; + } + } +} + +} +} +} \ No newline at end of file diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h new file mode 100644 index 0000000000..608ac22682 --- /dev/null +++ b/cpp/src/utils/ValidationUtil.h @@ -0,0 +1,20 @@ +#pragma once + +#include "Error.h" + +namespace zilliz { +namespace milvus { +namespace server { + +ServerError +ValidateTableName(const std::string& table_name); + +ServerError +ValidateTableDimension(int64_t dimension); + +ServerError +ValidateTableIndexType(int32_t index_type); + +} +} +} \ No newline at end of file diff --git a/cpp/src/wrapper/IndexBuilder.cpp b/cpp/src/wrapper/IndexBuilder.cpp index d4429c381a..41859907a7 100644 --- a/cpp/src/wrapper/IndexBuilder.cpp +++ b/cpp/src/wrapper/IndexBuilder.cpp @@ -71,7 +71,8 @@ Index_ptr IndexBuilder::build_all(const long &nb, { LOG(DEBUG) << "Build index by GPU"; // TODO: list support index-type. - faiss::Index *ori_index = faiss::index_factory(opd_->d, opd_->get_index_type(nb).c_str()); + faiss::MetricType metric_type = opd_->metric_type == "L2" ? faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + faiss::Index *ori_index = faiss::index_factory(opd_->d, opd_->get_index_type(nb).c_str(), metric_type); std::lock_guard lk(gpu_resource); faiss::gpu::StandardGpuResources res; @@ -90,7 +91,8 @@ Index_ptr IndexBuilder::build_all(const long &nb, #else { LOG(DEBUG) << "Build index by CPU"; - faiss::Index *index = faiss::index_factory(opd_->d, opd_->get_index_type(nb).c_str()); + faiss::MetricType metric_type = opd_->metric_type == "L2" ? faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + faiss::Index *index = faiss::index_factory(opd_->d, opd_->get_index_type(nb).c_str(), metric_type); if (!index->is_trained) { nt == 0 || xt == nullptr ? index->train(nb, xb) : index->train(nt, xt); @@ -113,7 +115,8 @@ BgCpuBuilder::BgCpuBuilder(const zilliz::milvus::engine::Operand_ptr &opd) : Ind Index_ptr BgCpuBuilder::build_all(const long &nb, const float *xb, const long *ids, const long &nt, const float *xt) { std::shared_ptr index = nullptr; - index.reset(faiss::index_factory(opd_->d, opd_->get_index_type(nb).c_str())); + faiss::MetricType metric_type = opd_->metric_type == "L2" ? faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + index.reset(faiss::index_factory(opd_->d, opd_->get_index_type(nb).c_str(), metric_type)); LOG(DEBUG) << "Build index by CPU"; { diff --git a/cpp/src/wrapper/Operand.cpp b/cpp/src/wrapper/Operand.cpp index 25341676a6..62a863ae0e 100644 --- a/cpp/src/wrapper/Operand.cpp +++ b/cpp/src/wrapper/Operand.cpp @@ -4,6 +4,7 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// +#include "src/server/ServerConfig.h" #include "Operand.h" @@ -16,12 +17,14 @@ using std::string; enum IndexType { Invalid_Option = 0, IVF = 1, - IDMAP = 2 + IDMAP = 2, + IVFSQ8 = 3, }; IndexType resolveIndexType(const string &index_type) { if (index_type == "IVF") { return IndexType::IVF; } if (index_type == "IDMap") { return IndexType::IDMAP; } + if (index_type == "IVFSQ8") { return IndexType::IVFSQ8; } return IndexType::Invalid_Option; } @@ -29,27 +32,44 @@ IndexType resolveIndexType(const string &index_type) { string Operand::get_index_type(const int &nb) { if (!index_str.empty()) { return index_str; } - // TODO: support OPQ or ... - if (!preproc.empty()) { index_str += (preproc + ","); } - switch (resolveIndexType(index_type)) { case Invalid_Option: { // TODO: add exception break; } case IVF: { + + using namespace zilliz::milvus::server; + ServerConfig &config = ServerConfig::GetInstance(); + ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); + size_t nlist = engine_config.GetInt32Value(CONFIG_NLIST, 16384); + index_str += (ncent != 0 ? index_type + std::to_string(ncent) : - index_type + std::to_string(int(nb / 1000000.0 * 16384))); + index_type + std::to_string(int(nb / 1000000.0 * nlist))); +// std::cout<<"nlist = "< -#include -#include -#include - #include "utils.h" #include "db/DB.h" #include "db/DBImpl.h" #include "db/MetaConsts.h" #include "db/Factories.h" +#include +#include + +#include + +#include +#include + using namespace zilliz::milvus; namespace { @@ -272,7 +275,9 @@ TEST_F(DBTest2, DELETE_TEST) { stat = db_->DescribeTable(table_info_get); ASSERT_STATS(stat); - ASSERT_TRUE(boost::filesystem::exists(table_info_get.location_)); + bool has_table = false; + db_->HasTable(TABLE_NAME, has_table); + ASSERT_TRUE(has_table); engine::IDNumbers vector_ids; @@ -293,5 +298,7 @@ TEST_F(DBTest2, DELETE_TEST) { stat = db_->DeleteTable(TABLE_NAME, dates); std::this_thread::sleep_for(std::chrono::seconds(2)); ASSERT_TRUE(stat.ok()); - ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); + + db_->HasTable(TABLE_NAME, has_table); + ASSERT_FALSE(has_table); }; diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp new file mode 100644 index 0000000000..2f6952c79d --- /dev/null +++ b/cpp/unittest/db/mem_test.cpp @@ -0,0 +1,372 @@ +#include "gtest/gtest.h" + +#include "db/VectorSource.h" +#include "db/MemTableFile.h" +#include "db/MemTable.h" +#include "utils.h" +#include "db/Factories.h" +#include "db/Constants.h" +#include "db/EngineFactory.h" +#include "metrics/Metrics.h" +#include "db/MetaConsts.h" +#include "boost/filesystem.hpp" + +#include +#include +#include +#include +#include + +using namespace zilliz::milvus; + +namespace { + +static const std::string TABLE_NAME = "test_group"; +static constexpr int64_t TABLE_DIM = 256; +static constexpr int64_t VECTOR_COUNT = 250000; +static constexpr int64_t INSERT_LOOP = 10000; + +engine::meta::TableSchema BuildTableSchema() { + engine::meta::TableSchema table_info; + table_info.dimension_ = TABLE_DIM; + table_info.table_id_ = TABLE_NAME; + table_info.engine_type_ = (int) engine::EngineType::FAISS_IDMAP; + return table_info; +} + +void BuildVectors(int64_t n, std::vector &vectors) { + vectors.clear(); + vectors.resize(n * TABLE_DIM); + float *data = vectors.data(); + for (int i = 0; i < n; i++) { + for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); + data[TABLE_DIM * i] += i / 2000.; + } +} +} + +TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) { + + std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + + engine::meta::TableSchema table_schema = BuildTableSchema(); + auto status = impl_->CreateTable(table_schema); + ASSERT_TRUE(status.ok()); + + engine::meta::TableFileSchema table_file_schema; + table_file_schema.table_id_ = TABLE_NAME; + status = impl_->CreateTableFile(table_file_schema); + ASSERT_TRUE(status.ok()); + + int64_t n = 100; + std::vector vectors; + BuildVectors(n, vectors); + + engine::VectorSource source(n, vectors.data()); + + size_t num_vectors_added; + engine::ExecutionEnginePtr execution_engine_ = engine::EngineFactory::Build(table_file_schema.dimension_, + table_file_schema.location_, + (engine::EngineType) table_file_schema.engine_type_); + status = source.Add(execution_engine_, table_file_schema, 50, num_vectors_added); + ASSERT_TRUE(status.ok()); + + ASSERT_EQ(num_vectors_added, 50); + + engine::IDNumbers vector_ids = source.GetVectorIds(); + ASSERT_EQ(vector_ids.size(), 50); + + status = source.Add(execution_engine_, table_file_schema, 60, num_vectors_added); + ASSERT_TRUE(status.ok()); + + ASSERT_EQ(num_vectors_added, 50); + + vector_ids = source.GetVectorIds(); + ASSERT_EQ(vector_ids.size(), 100); + + status = impl_->DropAll(); + ASSERT_TRUE(status.ok()); +} + +TEST_F(NewMemManagerTest, MEM_TABLE_FILE_TEST) { + + std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + auto options = engine::OptionsFactory::Build(); + + engine::meta::TableSchema table_schema = BuildTableSchema(); + auto status = impl_->CreateTable(table_schema); + ASSERT_TRUE(status.ok()); + + engine::MemTableFile mem_table_file(TABLE_NAME, impl_, options); + + int64_t n_100 = 100; + std::vector vectors_100; + BuildVectors(n_100, vectors_100); + + engine::VectorSource::Ptr source = std::make_shared(n_100, vectors_100.data()); + + status = mem_table_file.Add(source); + ASSERT_TRUE(status.ok()); + +// std::cout << mem_table_file.GetCurrentMem() << " " << mem_table_file.GetMemLeft() << std::endl; + + engine::IDNumbers vector_ids = source->GetVectorIds(); + ASSERT_EQ(vector_ids.size(), 100); + + size_t singleVectorMem = sizeof(float) * TABLE_DIM; + ASSERT_EQ(mem_table_file.GetCurrentMem(), n_100 * singleVectorMem); + + int64_t n_max = engine::MAX_TABLE_FILE_MEM / singleVectorMem; + std::vector vectors_128M; + BuildVectors(n_max, vectors_128M); + + engine::VectorSource::Ptr source_128M = std::make_shared(n_max, vectors_128M.data()); + status = mem_table_file.Add(source_128M); + + vector_ids = source_128M->GetVectorIds(); + ASSERT_EQ(vector_ids.size(), n_max - n_100); + + ASSERT_TRUE(mem_table_file.IsFull()); + + status = impl_->DropAll(); + ASSERT_TRUE(status.ok()); +} + +TEST_F(NewMemManagerTest, MEM_TABLE_TEST) { + + std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + auto options = engine::OptionsFactory::Build(); + + engine::meta::TableSchema table_schema = BuildTableSchema(); + auto status = impl_->CreateTable(table_schema); + ASSERT_TRUE(status.ok()); + + int64_t n_100 = 100; + std::vector vectors_100; + BuildVectors(n_100, vectors_100); + + engine::VectorSource::Ptr source_100 = std::make_shared(n_100, vectors_100.data()); + + engine::MemTable mem_table(TABLE_NAME, impl_, options); + + status = mem_table.Add(source_100); + ASSERT_TRUE(status.ok()); + engine::IDNumbers vector_ids = source_100->GetVectorIds(); + ASSERT_EQ(vector_ids.size(), 100); + + engine::MemTableFile::Ptr mem_table_file; + mem_table.GetCurrentMemTableFile(mem_table_file); + size_t singleVectorMem = sizeof(float) * TABLE_DIM; + ASSERT_EQ(mem_table_file->GetCurrentMem(), n_100 * singleVectorMem); + + int64_t n_max = engine::MAX_TABLE_FILE_MEM / singleVectorMem; + std::vector vectors_128M; + BuildVectors(n_max, vectors_128M); + + engine::VectorSource::Ptr source_128M = std::make_shared(n_max, vectors_128M.data()); + status = mem_table.Add(source_128M); + ASSERT_TRUE(status.ok()); + + vector_ids = source_128M->GetVectorIds(); + ASSERT_EQ(vector_ids.size(), n_max); + + mem_table.GetCurrentMemTableFile(mem_table_file); + ASSERT_EQ(mem_table_file->GetCurrentMem(), n_100 * singleVectorMem); + + ASSERT_EQ(mem_table.GetTableFileCount(), 2); + + int64_t n_1G = 1024000; + std::vector vectors_1G; + BuildVectors(n_1G, vectors_1G); + + engine::VectorSource::Ptr source_1G = std::make_shared(n_1G, vectors_1G.data()); + + status = mem_table.Add(source_1G); + ASSERT_TRUE(status.ok()); + + vector_ids = source_1G->GetVectorIds(); + ASSERT_EQ(vector_ids.size(), n_1G); + + int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / engine::MAX_TABLE_FILE_MEM); + ASSERT_EQ(mem_table.GetTableFileCount(), expectedTableFileCount); + + status = mem_table.Serialize(); + ASSERT_TRUE(status.ok()); + + status = impl_->DropAll(); + ASSERT_TRUE(status.ok()); +} + +TEST_F(NewMemManagerTest, SERIAL_INSERT_SEARCH_TEST) { + + auto options = engine::OptionsFactory::Build(); + options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + std::map> search_vectors; + { + engine::IDNumbers vector_ids; + int64_t nb = 1024000; + std::vector xb; + BuildVectors(nb, xb); + engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + ASSERT_TRUE(status.ok()); + + std::this_thread::sleep_for(std::chrono::seconds(3)); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, nb - 1); + + int64_t num_query = 20; + for (int64_t i = 0; i < num_query; ++i) { + int64_t index = dis(gen); + std::vector search; + for (int64_t j = 0; j < TABLE_DIM; j++) { + search.push_back(xb[index * TABLE_DIM + j]); + } + search_vectors.insert(std::make_pair(vector_ids[index], search)); + } + } + + int k = 10; + for (auto &pair : search_vectors) { + auto &search = pair.second; + engine::QueryResults results; + stat = db_->Query(TABLE_NAME, k, 1, search.data(), results); + ASSERT_EQ(results[0][0].first, pair.first); + ASSERT_LT(results[0][0].second, 0.00001); + } + + delete db_; + boost::filesystem::remove_all(options.meta.path); + +} + +TEST_F(NewMemManagerTest, INSERT_TEST) { + + auto options = engine::OptionsFactory::Build(); + options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + auto start_time = METRICS_NOW_TIME; + + int insert_loop = 20; + for (int i = 0; i < insert_loop; ++i) { + int64_t nb = 409600; + std::vector xb; + BuildVectors(nb, xb); + engine::IDNumbers vector_ids; + engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + ASSERT_TRUE(status.ok()); + } + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + LOG(DEBUG) << "total_time spent in INSERT_TEST (ms) : " << total_time; + + delete db_; + boost::filesystem::remove_all(options.meta.path); + +} + +TEST_F(NewMemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { + + auto options = engine::OptionsFactory::Build(); + options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + engine::IDNumbers vector_ids; + engine::IDNumbers target_ids; + + int64_t nb = 409600; + std::vector xb; + BuildVectors(nb, xb); + + int64_t qb = 5; + std::vector qxb; + BuildVectors(qb, qxb); + + std::thread search([&]() { + engine::QueryResults results; + int k = 10; + std::this_thread::sleep_for(std::chrono::seconds(2)); + + INIT_TIMER; + std::stringstream ss; + uint64_t count = 0; + uint64_t prev_count = 0; + + for (auto j = 0; j < 10; ++j) { + ss.str(""); + db_->Size(count); + prev_count = count; + + START_TIMER; + stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); + ss << "Search " << j << " With Size " << count / engine::meta::M << " M"; + STOP_TIMER(ss.str()); + + ASSERT_STATS(stat); + for (auto k = 0; k < qb; ++k) { + ASSERT_EQ(results[k][0].first, target_ids[k]); + ss.str(""); + ss << "Result [" << k << "]:"; + for (auto result : results[k]) { + ss << result.first << " "; + } + /* LOG(DEBUG) << ss.str(); */ + } + ASSERT_TRUE(count >= prev_count); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + }); + + int loop = 20; + + for (auto i = 0; i < loop; ++i) { + if (i == 0) { + db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); + ASSERT_EQ(target_ids.size(), qb); + } else { + db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + } + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + search.join(); + + delete db_; + boost::filesystem::remove_all(options.meta.path); + +}; + diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 0f3a92af09..5bce4058b1 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -38,7 +38,7 @@ TEST_F(MetaTest, TABLE_TEST) { table.table_id_ = table_id; status = impl_->CreateTable(table); - ASSERT_TRUE(status.ok()); + ASSERT_TRUE(status.IsAlreadyExist()); table.table_id_ = ""; status = impl_->CreateTable(table); diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 7fdb30a204..9bf4e6d19a 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -3,17 +3,19 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// -#include -#include -#include -#include - #include "utils.h" #include "db/DB.h" #include "db/DBImpl.h" #include "db/MetaConsts.h" #include "db/Factories.h" +#include +#include +#include + +#include +#include + using namespace zilliz::milvus; namespace { @@ -271,8 +273,9 @@ TEST_F(MySQLDBTest, DELETE_TEST) { stat = db_->DescribeTable(table_info_get); ASSERT_STATS(stat); -// std::cout << "location: " << table_info_get.location_ << std::endl; - ASSERT_TRUE(boost::filesystem::exists(table_info_get.location_)); + bool has_table = false; + db_->HasTable(TABLE_NAME, has_table); + ASSERT_TRUE(has_table); engine::IDNumbers vector_ids; @@ -295,7 +298,9 @@ TEST_F(MySQLDBTest, DELETE_TEST) { std::this_thread::sleep_for(std::chrono::seconds(5)); // std::cout << "5 sec finish" << std::endl; ASSERT_TRUE(stat.ok()); -// ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); + + db_->HasTable(TABLE_NAME, has_table); + ASSERT_FALSE(has_table); delete db_; diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index db10bcbadf..c4ee7a48b5 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -6,7 +6,7 @@ #include #include "db/scheduler/task/SearchTask.h" - +#include #include using namespace zilliz::milvus; @@ -73,13 +73,13 @@ TEST(DBSearchTest, TOPK_TEST) { ASSERT_EQ(src_result.size(), NQ); engine::SearchContext::ResultSet target_result; - status = engine::SearchTask::TopkResult(target_result, TOP_K, target_result); + status = engine::SearchTask::TopkResult(target_result, TOP_K, true, target_result); ASSERT_TRUE(status.ok()); - status = engine::SearchTask::TopkResult(target_result, TOP_K, src_result); + status = engine::SearchTask::TopkResult(target_result, TOP_K, true, src_result); ASSERT_FALSE(status.ok()); - status = engine::SearchTask::TopkResult(src_result, TOP_K, target_result); + status = engine::SearchTask::TopkResult(src_result, TOP_K, true, target_result); ASSERT_TRUE(status.ok()); ASSERT_TRUE(src_result.empty()); ASSERT_EQ(target_result.size(), NQ); @@ -92,7 +92,7 @@ TEST(DBSearchTest, TOPK_TEST) { status = engine::SearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result); ASSERT_TRUE(status.ok()); - status = engine::SearchTask::TopkResult(src_result, TOP_K, target_result); + status = engine::SearchTask::TopkResult(src_result, TOP_K, true, target_result); ASSERT_TRUE(status.ok()); for(uint64_t i = 0; i < NQ; i++) { ASSERT_EQ(target_result[i].size(), TOP_K); @@ -101,7 +101,7 @@ TEST(DBSearchTest, TOPK_TEST) { wrong_topk = TOP_K + 10; BuildResult(NQ, wrong_topk, src_ids, src_distence); - status = engine::SearchTask::TopkResult(src_result, TOP_K, target_result); + status = engine::SearchTask::TopkResult(src_result, TOP_K, true, target_result); ASSERT_TRUE(status.ok()); for(uint64_t i = 0; i < NQ; i++) { ASSERT_EQ(target_result[i].size(), TOP_K); @@ -126,7 +126,7 @@ TEST(DBSearchTest, MERGE_TEST) { { engine::SearchContext::Id2DistanceMap src = src_result[0]; engine::SearchContext::Id2DistanceMap target = target_result[0]; - status = engine::SearchTask::MergeResult(src, target, 10); + status = engine::SearchTask::MergeResult(src, target, 10, true); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), 10); CheckResult(src_result[0], target_result[0], target); @@ -135,7 +135,7 @@ TEST(DBSearchTest, MERGE_TEST) { { engine::SearchContext::Id2DistanceMap src = src_result[0]; engine::SearchContext::Id2DistanceMap target; - status = engine::SearchTask::MergeResult(src, target, 10); + status = engine::SearchTask::MergeResult(src, target, 10, true); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count); ASSERT_TRUE(src.empty()); @@ -145,7 +145,7 @@ TEST(DBSearchTest, MERGE_TEST) { { engine::SearchContext::Id2DistanceMap src = src_result[0]; engine::SearchContext::Id2DistanceMap target = target_result[0]; - status = engine::SearchTask::MergeResult(src, target, 30); + status = engine::SearchTask::MergeResult(src, target, 30, true); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count + target_count); CheckResult(src_result[0], target_result[0], target); @@ -154,9 +154,9 @@ TEST(DBSearchTest, MERGE_TEST) { { engine::SearchContext::Id2DistanceMap target = src_result[0]; engine::SearchContext::Id2DistanceMap src = target_result[0]; - status = engine::SearchTask::MergeResult(src, target, 30); + status = engine::SearchTask::MergeResult(src, target, 30, true); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count + target_count); CheckResult(src_result[0], target_result[0], target); } -} \ No newline at end of file +} diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 70c0712549..ae05c59d3b 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -106,6 +106,18 @@ zilliz::milvus::engine::Options MySQLDBTest::GetOptions() { return options; } +void NewMemManagerTest::InitLog() { + el::Configurations defaultConf; + defaultConf.setToDefault(); + defaultConf.set(el::Level::Debug, + el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)"); + el::Loggers::reconfigureLogger("default", defaultConf); +} + +void NewMemManagerTest::SetUp() { + InitLog(); +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); if (argc > 1) { diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index 361c24b4be..9c126030c2 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -30,7 +30,7 @@ #define STOP_TIMER(name) #endif -void ASSERT_STATS(zilliz::milvus::engine::Status& stat); +void ASSERT_STATS(zilliz::milvus::engine::Status &stat); //class TestEnv : public ::testing::Environment { //public: @@ -54,8 +54,8 @@ void ASSERT_STATS(zilliz::milvus::engine::Status& stat); // ::testing::AddGlobalTestEnvironment(new TestEnv); class DBTest : public ::testing::Test { -protected: - zilliz::milvus::engine::DB* db_; + protected: + zilliz::milvus::engine::DB *db_; void InitLog(); virtual void SetUp() override; @@ -64,13 +64,13 @@ protected: }; class DBTest2 : public DBTest { -protected: + protected: virtual zilliz::milvus::engine::Options GetOptions() override; }; class MetaTest : public DBTest { -protected: + protected: std::shared_ptr impl_; virtual void SetUp() override; @@ -78,12 +78,17 @@ protected: }; class MySQLTest : public ::testing::Test { -protected: + protected: // std::shared_ptr impl_; zilliz::milvus::engine::DBMetaOptions getDBMetaOptions(); }; -class MySQLDBTest : public ::testing::Test { -protected: +class MySQLDBTest : public ::testing::Test { + protected: zilliz::milvus::engine::Options GetOptions(); }; + +class NewMemManagerTest : public ::testing::Test { + void InitLog(); + void SetUp() override; +}; diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index f044df8d8c..ff2535e3fd 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -22,13 +22,13 @@ add_executable(wrapper_test ${wrapper_test_src}) set(wrapper_libs stdc++ - boost_system - boost_filesystem + boost_system_static + boost_filesystem_static libgpufaiss.a faiss cudart cublas - sqlite3 + sqlite snappy bz2 z diff --git a/cpp/unittest/faiss_wrapper/wrapper_test.cpp b/cpp/unittest/faiss_wrapper/wrapper_test.cpp index 67a6c3cde8..6f4a651a55 100644 --- a/cpp/unittest/faiss_wrapper/wrapper_test.cpp +++ b/cpp/unittest/faiss_wrapper/wrapper_test.cpp @@ -4,12 +4,15 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// -#include + #include "wrapper/Operand.h" #include "wrapper/Index.h" #include "wrapper/IndexBuilder.h" +#include +#include + using namespace zilliz::milvus::engine; diff --git a/cpp/unittest/license/CMakeLists.txt b/cpp/unittest/license/CMakeLists.txt index 93af82ed56..39d0ede197 100644 --- a/cpp/unittest/license/CMakeLists.txt +++ b/cpp/unittest/license/CMakeLists.txt @@ -33,11 +33,11 @@ set(db_libs nvidia-ml cudart cublas - boost_system - boost_filesystem + boost_system_static + boost_filesystem_static lz4 crypto - boost_serialization + boost_serialization_static ) target_link_libraries(license_test ${db_libs} ${unittest_libs}) diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index d31e44c056..513b37b8c2 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -66,9 +66,9 @@ target_link_libraries(metrics_test faiss cudart cublas - sqlite3 - boost_system - boost_filesystem + sqlite + boost_system_static + boost_filesystem_static lz4 metrics gtest diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index c4112cda9e..e49aed500c 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -37,9 +37,9 @@ set(require_libs faiss cudart cublas - sqlite3 - boost_system - boost_filesystem + sqlite + boost_system_static + boost_filesystem_static snappy z bz2 diff --git a/cpp/unittest/storage/CMakeLists.txt b/cpp/unittest/storage/CMakeLists.txt index 6b4303b70a..d4deaefab8 100644 --- a/cpp/unittest/storage/CMakeLists.txt +++ b/cpp/unittest/storage/CMakeLists.txt @@ -25,7 +25,7 @@ set(s3_client_libs stdc++ aws-cpp-sdk-s3 aws-cpp-sdk-core - boost_filesystem + boost_filesystem_static ) target_link_libraries(s3_test ${s3_client_libs} diff --git a/cpp/unittest/utils/CMakeLists.txt b/cpp/unittest/utils/CMakeLists.txt new file mode 100644 index 0000000000..61e4459462 --- /dev/null +++ b/cpp/unittest/utils/CMakeLists.txt @@ -0,0 +1,30 @@ +#------------------------------------------------------------------------------- +# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +# Unauthorized copying of this file, via any medium is strictly prohibited. +# Proprietary and confidential. +#------------------------------------------------------------------------------- + +# Make sure that your call to link_directories takes place before your call to the relevant add_executable. +include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") +link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") + +set(validation_util_src + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.h) + +set(validation_util_test_src + ${unittest_srcs} + ${validation_util_src} + ${require_files} + ValidationUtilTest.cpp + ) + +add_executable(valication_util_test + ${validation_util_test_src} + ${config_files}) + +target_link_libraries(valication_util_test + ${unittest_libs} + boost_filesystem_static) + +install(TARGETS valication_util_test DESTINATION bin) \ No newline at end of file diff --git a/cpp/unittest/utils/ValidationUtilTest.cpp b/cpp/unittest/utils/ValidationUtilTest.cpp new file mode 100644 index 0000000000..095614e325 --- /dev/null +++ b/cpp/unittest/utils/ValidationUtilTest.cpp @@ -0,0 +1,61 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include + +#include "utils/ValidationUtil.h" +#include "utils/Error.h" + +#include + +using namespace zilliz::milvus::server; + +TEST(ValidationUtilTest, TableNameTest) { + std::string table_name = "Normal123_"; + ServerError res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_SUCCESS); + + table_name = "12sds"; + res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + + table_name = ""; + res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + + table_name = "_asdasd"; + res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_SUCCESS); + + table_name = "!@#!@"; + res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + + table_name = "中文"; + res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + + + table_name = std::string('a', 32768); + res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); +} + + +TEST(ValidationUtilTest, TableDimensionTest) { + ASSERT_EQ(ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ValidateTableDimension(16384), SERVER_SUCCESS); + ASSERT_EQ(ValidateTableDimension(1), SERVER_SUCCESS); +} + +TEST(ValidationUtilTest, TableIndexTypeTest) { + ASSERT_EQ(ValidateTableIndexType(0), SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(ValidateTableIndexType(1), SERVER_SUCCESS); + ASSERT_EQ(ValidateTableIndexType(2), SERVER_SUCCESS); + ASSERT_EQ(ValidateTableIndexType(3), SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(ValidateTableIndexType(4), SERVER_INVALID_INDEX_TYPE); +}