diff --git a/CHANGELOG.md b/CHANGELOG.md index 07ab912c0a..53e427dcb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,9 @@ Please mark all change in change log and use the ticket from JIRA. - \#412 - Message returned is confused when partition created with null partition name - \#416 - Drop the same partition success repeatally - \#440 - Query API in customization still uses old version +- \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 +- \#486 - gpu no usage during index building ## Feature - \#12 - Pure CPU version for Milvus @@ -27,6 +29,8 @@ Please mark all change in change log and use the ticket from JIRA. - \#226 - Experimental shards middleware for Milvus - \#227 - Support new index types SPTAG-KDT and SPTAG-BKT - \#346 - Support build index with multiple gpu +- \#488 - Add log in scheduler/optimizer +- \#502 - C++ SDK support IVFPQ and SPTAG ## Improvement - \#255 - Add ivfsq8 test report detailed version @@ -43,6 +47,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#409 - Add a Fallback pass in optimizer - \#433 - C++ SDK query result is not easy to use - \#449 - Add ShowPartitions example for C++ SDK +- \#470 - Small raw files should not be build index ## Task diff --git a/README_CN.md b/README_CN.md index 374cefa9bd..b101ea0570 100644 --- a/README_CN.md +++ b/README_CN.md @@ -15,7 +15,7 @@ Milvus 是一款开源的、针对海量特征向量的相似性搜索引擎。 若要了解 Milvus 详细介绍和整体架构，请访问 [Milvus 简介](https://www.milvus.io/docs/zh-CN/aboutmilvus/overview/)。 -Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java) 以及 C++ 的 API 接口。 +Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java) 以及[C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) 的 API 接口。 通过 [版本发布说明](https://milvus.io/docs/zh-CN/release/v0.5.3/) 获取最新版本的功能和更新。 diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 01048bd953..0eed167fb4 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -114,7 +114,8 @@ pipeline { stage("Deploy to Development") { environment { - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() + FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() } agent { @@ -248,7 +249,8 @@ pipeline { stage("Deploy to Development") { environment { - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() + FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() } agent { diff --git a/ci/jenkins/step/package.groovy b/ci/jenkins/step/package.groovy index c07dcd9d88..35d2d8cfdf 100644 --- a/ci/jenkins/step/package.groovy +++ b/ci/jenkins/step/package.groovy @@ -2,7 +2,9 @@ timeout(time: 5, unit: 'MINUTES') { dir ("ci/jenkins/scripts") { sh "pip3 install -r requirements.txt" sh "./yaml_processor.py merge -f /opt/milvus/conf/server_config.yaml -m ../yaml/update_server_config.yaml -i && rm /opt/milvus/conf/server_config.yaml.bak" + sh "sed -i 's/\\/tmp\\/milvus/\\/opt\\/milvus/g' /opt/milvus/conf/log_config.conf" } + sh "rm -rf /opt/milvus/unittest" sh "tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz -C /opt/ milvus" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz") diff --git a/ci/jenkinsfile/cleanup_staging.groovy b/ci/jenkinsfile/cleanup_staging.groovy deleted file mode 100644 index 2e9332fa6e..0000000000 --- a/ci/jenkinsfile/cleanup_staging.groovy +++ /dev/null @@ -1,13 +0,0 @@ -try { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - } -} catch (exc) { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - } - throw exc -} - diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy deleted file mode 100644 index e57988fefe..0000000000 --- a/ci/jenkinsfile/cluster_cleanup_dev.groovy +++ /dev/null @@ -1,13 +0,0 @@ -try { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - } -} catch (exc) { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - } - throw exc -} - diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy deleted file mode 100644 index 7f2a584256..0000000000 --- a/ci/jenkinsfile/cluster_deploy2dev.groovy +++ /dev/null @@ -1,24 +0,0 @@ -try { - sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' - sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' - sh 'helm repo update' - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("milvus/milvus-cluster") { - sh "helm install --wait --timeout 300 --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.5.0 . " - } - } - /* - timeout(time: 2, unit: 'MINUTES') { - waitUntil { - def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true - return !result - } - } - */ -} catch (exc) { - echo 'Helm running failed!' - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - throw exc -} - diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy deleted file mode 100644 index 4a15b926cf..0000000000 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ /dev/null @@ -1,12 +0,0 @@ -timeout(time: 25, unit: 'MINUTES') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements_cluster.txt' - sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" - } - } catch (exc) { - echo 'Milvus Cluster Test Failed !' - throw exc - } -} diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy deleted file mode 100644 index b00c8fa335..0000000000 --- a/ci/jenkinsfile/deploy2dev.groovy +++ /dev/null @@ -1,16 +0,0 @@ -try { - sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' - sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' - sh 'helm repo update' - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ." - } - } -} catch (exc) { - echo 'Helm running failed!' - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - throw exc -} - diff --git a/ci/jenkinsfile/deploy2staging.groovy b/ci/jenkinsfile/deploy2staging.groovy deleted file mode 100644 index 42ccfda71a..0000000000 --- a/ci/jenkinsfile/deploy2staging.groovy +++ /dev/null @@ -1,16 +0,0 @@ -try { - sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' - sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' - sh 'helm repo update' - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ." - } - } -} catch (exc) { - echo 'Helm running failed!' - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - throw exc -} - diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy deleted file mode 100644 index 9b265ac401..0000000000 --- a/ci/jenkinsfile/dev_test.groovy +++ /dev/null @@ -1,28 +0,0 @@ -timeout(time: 30, unit: 'MINUTES') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal=true" - } - // mysql database backend test - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - - if (!fileExists('milvus-helm')) { - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - } - } - dir ("milvus-helm") { - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ." - } - } - dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal=true" - } - } catch (exc) { - echo 'Milvus Test Failed !' - throw exc - } -} diff --git a/ci/jenkinsfile/dev_test_all.groovy b/ci/jenkinsfile/dev_test_all.groovy deleted file mode 100644 index b82d995d8c..0000000000 --- a/ci/jenkinsfile/dev_test_all.groovy +++ /dev/null @@ -1,29 +0,0 @@ -timeout(time: 60, unit: 'MINUTES') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal=true" - } - - // mysql database backend test - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - - if (!fileExists('milvus-helm')) { - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - } - } - dir ("milvus-helm") { - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ." - } - } - dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal=true" - } - } catch (exc) { - echo 'Milvus Test Failed !' - throw exc - } -} diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy deleted file mode 100644 index 27e50c64c4..0000000000 --- a/ci/jenkinsfile/milvus_build.groovy +++ /dev/null @@ -1,30 +0,0 @@ -container('milvus-build-env') { - timeout(time: 120, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Build Engine') { - dir ("milvus_engine") { - try { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - - dir ("core") { - sh "git config --global user.email \"test@zilliz.com\"" - sh "git config --global user.name \"test\"" - withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "./build.sh -l" - sh "rm -rf cmake_build" - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \ - && export JFROG_USER_NAME='${USERNAME}' \ - && export JFROG_PASSWORD='${PASSWORD}' \ - && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \ - && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u -c" - - sh "./coverage.sh -u root -p 123456 -t \$POD_IP" - } - } - } catch (exc) { - updateGitlabCommitStatus name: 'Build Engine', state: 'failed' - throw exc - } - } - } - } -} diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy deleted file mode 100644 index 3f221b8947..0000000000 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ /dev/null @@ -1,28 +0,0 @@ -container('milvus-build-env') { - timeout(time: 120, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Build Engine') { - dir ("milvus_engine") { - try { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - - dir ("core") { - sh "git config --global user.email \"test@zilliz.com\"" - sh "git config --global user.name \"test\"" - withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "./build.sh -l" - sh "rm -rf cmake_build" - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \ - && export JFROG_USER_NAME='${USERNAME}' \ - && export JFROG_PASSWORD='${PASSWORD}' \ - && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \ - && ./build.sh -t ${params.BUILD_TYPE} -j -d /opt/milvus" - } - } - } catch (exc) { - updateGitlabCommitStatus name: 'Build Engine', state: 'failed' - throw exc - } - } - } - } -} diff --git a/ci/jenkinsfile/nightly_publish_docker.groovy b/ci/jenkinsfile/nightly_publish_docker.groovy deleted file mode 100644 index 8c6121bec8..0000000000 --- a/ci/jenkinsfile/nightly_publish_docker.groovy +++ /dev/null @@ -1,38 +0,0 @@ -container('publish-docker') { - timeout(time: 15, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Publish Engine Docker') { - try { - dir ("${PROJECT_NAME}_build") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("docker/deploy/ubuntu16.04/free_version") { - sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" - sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" - try { - def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}") - docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_PUBLISH_USER}") { - customImage.push() - } - docker.withRegistry('https://zilliz.azurecr.cn', "${params.AZURE_DOCKER_PUBLISH_USER}") { - customImage.push() - } - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'success' - echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled' - throw exc - } finally { - sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}" - } - } - } - } catch (exc) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed' - echo 'Publish docker failed!' - throw exc - } - } - } -} - diff --git a/ci/jenkinsfile/packaged_milvus.groovy b/ci/jenkinsfile/packaged_milvus.groovy deleted file mode 100644 index 1d30e21910..0000000000 --- a/ci/jenkinsfile/packaged_milvus.groovy +++ /dev/null @@ -1,44 +0,0 @@ -container('milvus-build-env') { - timeout(time: 5, unit: 'MINUTES') { - dir ("milvus_engine") { - dir ("core") { - gitlabCommitStatus(name: 'Packaged Engine') { - if (fileExists('milvus')) { - try { - sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\"" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' - throw exc - } - } else { - updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' - error("Milvus binary directory don't exists!") - } - } - - gitlabCommitStatus(name: 'Packaged Engine lcov') { - if (fileExists('lcov_out')) { - try { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("lcov_out/", "${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus lcov out Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}/lcov_out/\"" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed' - throw exc - } - } else { - updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed' - error("Milvus lcov out directory don't exists!") - } - } - } - } - } -} diff --git a/ci/jenkinsfile/packaged_milvus_no_ut.groovy b/ci/jenkinsfile/packaged_milvus_no_ut.groovy deleted file mode 100644 index bc68be374a..0000000000 --- a/ci/jenkinsfile/packaged_milvus_no_ut.groovy +++ /dev/null @@ -1,26 +0,0 @@ -container('milvus-build-env') { - timeout(time: 5, unit: 'MINUTES') { - dir ("milvus_engine") { - dir ("core") { - gitlabCommitStatus(name: 'Packaged Engine') { - if (fileExists('milvus')) { - try { - sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\"" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' - throw exc - } - } else { - updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' - error("Milvus binary directory don't exists!") - } - } - } - } - } -} diff --git a/ci/jenkinsfile/publish_docker.groovy b/ci/jenkinsfile/publish_docker.groovy deleted file mode 100644 index ef31eba9a4..0000000000 --- a/ci/jenkinsfile/publish_docker.groovy +++ /dev/null @@ -1,35 +0,0 @@ -container('publish-docker') { - timeout(time: 15, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Publish Engine Docker') { - try { - dir ("${PROJECT_NAME}_build") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("docker/deploy/ubuntu16.04/free_version") { - sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" - sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" - try { - def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}") - docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_PUBLISH_USER}") { - customImage.push() - } - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'success' - echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled' - throw exc - } finally { - sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}" - } - } - } - } catch (exc) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed' - echo 'Publish docker failed!' - throw exc - } - } - } -} - diff --git a/ci/jenkinsfile/staging_test.groovy b/ci/jenkinsfile/staging_test.groovy deleted file mode 100644 index dcf1787103..0000000000 --- a/ci/jenkinsfile/staging_test.groovy +++ /dev/null @@ -1,31 +0,0 @@ -timeout(time: 40, unit: 'MINUTES') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt' - def service_ip = sh (script: "kubectl get svc --namespace milvus-1 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim() - sh "pytest . --alluredir=\"test_out/staging/single/sqlite\" --ip ${service_ip}" - } - - // mysql database backend test - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" - - if (!fileExists('milvus-helm')) { - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - } - } - dir ("milvus-helm") { - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ." - } - } - dir ("${PROJECT_NAME}_test") { - def service_ip = sh (script: "kubectl get svc --namespace milvus-2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim() - sh "pytest . --alluredir=\"test_out/staging/single/mysql\" --ip ${service_ip}" - } - } catch (exc) { - echo 'Milvus Test Failed !' - throw exc - } -} diff --git a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy deleted file mode 100644 index 6bbd8a649f..0000000000 --- a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy +++ /dev/null @@ -1,14 +0,0 @@ -timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - if (fileExists('cluster_test_out')) { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("cluster_test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } else { - error("Milvus Dev Test Out directory don't exists!") - } - } -} - diff --git a/ci/jenkinsfile/upload_dev_test_out.groovy b/ci/jenkinsfile/upload_dev_test_out.groovy deleted file mode 100644 index 017b887334..0000000000 --- a/ci/jenkinsfile/upload_dev_test_out.groovy +++ /dev/null @@ -1,13 +0,0 @@ -timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - if (fileExists('test_out/dev')) { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("test_out/dev/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } else { - error("Milvus Dev Test Out directory don't exists!") - } - } -} diff --git a/ci/jenkinsfile/upload_staging_test_out.groovy b/ci/jenkinsfile/upload_staging_test_out.groovy deleted file mode 100644 index 1f1e66ab1b..0000000000 --- a/ci/jenkinsfile/upload_staging_test_out.groovy +++ /dev/null @@ -1,13 +0,0 @@ -timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - if (fileExists('test_out/staging')) { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("test_out/staging/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } else { - error("Milvus Dev Test Out directory don't exists!") - } - } -} diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile deleted file mode 100644 index 0c3fc32e5b..0000000000 --- a/ci/main_jenkinsfile +++ /dev/null @@ -1,396 +0,0 @@ -pipeline { - agent none - - options { - timestamps() - } - - environment { - PROJECT_NAME = "milvus" - LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() - SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" - GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" - SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" - DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" - } - - stages { - stage("Ubuntu 16.04") { - environment { - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' - ]); - - DOCKER_VERSION = VersionNumber([ - versionNumberString : '${DOCKER_VERSION_STR}' - ]); - } - - stages { - stage("Run Build") { - agent { - kubernetes { - cloud 'build-kubernetes' - label 'build' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - name: milvus-build-env - labels: - app: milvus - componet: build-env -spec: - containers: - - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.13 - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - command: - - cat - tty: true - resources: - limits: - memory: "28Gi" - cpu: "10.0" - nvidia.com/gpu: 1 - requests: - memory: "14Gi" - cpu: "5.0" - - name: milvus-mysql - image: mysql:5.6 - env: - - name: MYSQL_ROOT_PASSWORD - value: 123456 - ports: - - containerPort: 3306 - name: mysql -""" - } - } - stages { - stage('Build') { - steps { - gitlabCommitStatus(name: 'Build') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Build', state: 'canceled' - echo "Milvus Build aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Build', state: 'failed' - echo "Milvus Build failure !" - } - } - } - } - - stage("Publish docker and helm") { - agent { - kubernetes { - label 'publish' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-docker - image: registry.zilliz.com/library/zilliz_docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -""" - } - } - stages { - stage('Publish Docker') { - steps { - gitlabCommitStatus(name: 'Publish Docker') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' - echo "Milvus Publish Docker aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' - echo "Milvus Publish Docker failure !" - } - } - } - } - - stage("Deploy to Development") { - parallel { - stage("Single Node") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: test-cluster-config -""" - } - } - - stages { - stage("Deploy to Dev") { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } - } - } - } - } - stage("Dev Test") { - steps { - gitlabCommitStatus(name: 'Deloy Test') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" - } - } - } - } - } - stage ("Cleanup Dev") { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - } - } - } - post { - always { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - success { - script { - echo "Milvus Single Node CI/CD success !" - } - } - aborted { - script { - echo "Milvus Single Node CI/CD aborted !" - } - } - failure { - script { - echo "Milvus Single Node CI/CD failure !" - } - } - } - } - -// stage("Cluster") { -// agent { -// kubernetes { -// label 'dev-test' -// defaultContainer 'jnlp' -// yaml """ -// apiVersion: v1 -// kind: Pod -// metadata: -// labels: -// app: milvus -// componet: test -// spec: -// containers: -// - name: milvus-testframework -// image: registry.zilliz.com/milvus/milvus-test:v0.2 -// command: -// - cat -// tty: true -// volumeMounts: -// - name: kubeconf -// mountPath: /root/.kube/ -// readOnly: true -// volumes: -// - name: kubeconf -// secret: -// secretName: test-cluster-config -// """ -// } -// } -// stages { -// stage("Deploy to Dev") { -// steps { -// gitlabCommitStatus(name: 'Deloy to Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" -// } -// } -// } -// } -// } -// stage("Dev Test") { -// steps { -// gitlabCommitStatus(name: 'Deloy Test') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" -// load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" -// } -// } -// } -// } -// } -// stage ("Cleanup Dev") { -// steps { -// gitlabCommitStatus(name: 'Cleanup Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// } -// } -// } -// post { -// always { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// success { -// script { -// echo "Milvus Cluster CI/CD success !" -// } -// } -// aborted { -// script { -// echo "Milvus Cluster CI/CD aborted !" -// } -// } -// failure { -// script { -// echo "Milvus Cluster CI/CD failure !" -// } -// } -// } -// } - } - } - } - } - } - - post { - always { - script { - if (env.gitlabAfter != null) { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } - } - } - } - - success { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'success' - echo "Milvus CI/CD success !" - } - } - - aborted { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' - echo "Milvus CI/CD aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'failed' - echo "Milvus CI/CD failure !" - } - } - } -} - diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut deleted file mode 100644 index e8d7dae75a..0000000000 --- a/ci/main_jenkinsfile_no_ut +++ /dev/null @@ -1,396 +0,0 @@ -pipeline { - agent none - - options { - timestamps() - } - - environment { - PROJECT_NAME = "milvus" - LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() - SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" - GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" - SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" - DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" - } - - stages { - stage("Ubuntu 16.04") { - environment { - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' - ]); - - DOCKER_VERSION = VersionNumber([ - versionNumberString : '${DOCKER_VERSION_STR}' - ]); - } - - stages { - stage("Run Build") { - agent { - kubernetes { - cloud 'build-kubernetes' - label 'build' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - name: milvus-build-env - labels: - app: milvus - componet: build-env -spec: - containers: - - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.13 - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - command: - - cat - tty: true - resources: - limits: - memory: "28Gi" - cpu: "10.0" - nvidia.com/gpu: 1 - requests: - memory: "14Gi" - cpu: "5.0" - - name: milvus-mysql - image: mysql:5.6 - env: - - name: MYSQL_ROOT_PASSWORD - value: 123456 - ports: - - containerPort: 3306 - name: mysql -""" - } - } - stages { - stage('Build') { - steps { - gitlabCommitStatus(name: 'Build') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build_no_ut.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus_no_ut.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Build', state: 'canceled' - echo "Milvus Build aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Build', state: 'failed' - echo "Milvus Build failure !" - } - } - } - } - - stage("Publish docker and helm") { - agent { - kubernetes { - label 'publish' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-docker - image: registry.zilliz.com/library/zilliz_docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -""" - } - } - stages { - stage('Publish Docker') { - steps { - gitlabCommitStatus(name: 'Publish Docker') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' - echo "Milvus Publish Docker aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' - echo "Milvus Publish Docker failure !" - } - } - } - } - - stage("Deploy to Development") { - parallel { - stage("Single Node") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: test-cluster-config -""" - } - } - - stages { - stage("Deploy to Dev") { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } - } - } - } - } - stage("Dev Test") { - steps { - gitlabCommitStatus(name: 'Deloy Test') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" - } - } - } - } - } - stage ("Cleanup Dev") { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - } - } - } - post { - always { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - success { - script { - echo "Milvus Single Node CI/CD success !" - } - } - aborted { - script { - echo "Milvus Single Node CI/CD aborted !" - } - } - failure { - script { - echo "Milvus Single Node CI/CD failure !" - } - } - } - } - -// stage("Cluster") { -// agent { -// kubernetes { -// label 'dev-test' -// defaultContainer 'jnlp' -// yaml """ -// apiVersion: v1 -// kind: Pod -// metadata: -// labels: -// app: milvus -// componet: test -// spec: -// containers: -// - name: milvus-testframework -// image: registry.zilliz.com/milvus/milvus-test:v0.2 -// command: -// - cat -// tty: true -// volumeMounts: -// - name: kubeconf -// mountPath: /root/.kube/ -// readOnly: true -// volumes: -// - name: kubeconf -// secret: -// secretName: test-cluster-config -// """ -// } -// } -// stages { -// stage("Deploy to Dev") { -// steps { -// gitlabCommitStatus(name: 'Deloy to Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" -// } -// } -// } -// } -// } -// stage("Dev Test") { -// steps { -// gitlabCommitStatus(name: 'Deloy Test') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" -// load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" -// } -// } -// } -// } -// } -// stage ("Cleanup Dev") { -// steps { -// gitlabCommitStatus(name: 'Cleanup Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// } -// } -// } -// post { -// always { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// success { -// script { -// echo "Milvus Cluster CI/CD success !" -// } -// } -// aborted { -// script { -// echo "Milvus Cluster CI/CD aborted !" -// } -// } -// failure { -// script { -// echo "Milvus Cluster CI/CD failure !" -// } -// } -// } -// } - } - } - } - } - } - - post { - always { - script { - if (env.gitlabAfter != null) { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } - } - } - } - - success { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'success' - echo "Milvus CI/CD success !" - } - } - - aborted { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' - echo "Milvus CI/CD aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'failed' - echo "Milvus CI/CD failure !" - } - } - } -} - diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile deleted file mode 100644 index add9e00fb4..0000000000 --- a/ci/nightly_main_jenkinsfile +++ /dev/null @@ -1,478 +0,0 @@ -pipeline { - agent none - - options { - timestamps() - } - - environment { - PROJECT_NAME = "milvus" - LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() - SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" - GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" - SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" - DOCKER_VERSION_STR = "${env.gitlabAfter == null ? '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, \"yyyyMMdd\"}' : '${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}'}" - } - - stages { - stage("Ubuntu 16.04") { - environment { - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' - ]); - - DOCKER_VERSION = VersionNumber([ - versionNumberString : '${DOCKER_VERSION_STR}' - ]); - } - - stages { - stage("Run Build") { - agent { - kubernetes { - cloud 'build-kubernetes' - label 'build' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - name: milvus-build-env - labels: - app: milvus - componet: build-env -spec: - containers: - - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.13 - command: - - cat - tty: true - resources: - limits: - memory: "28Gi" - cpu: "10.0" - nvidia.com/gpu: 1 - requests: - memory: "14Gi" - cpu: "5.0" -""" - } - } - stages { - stage('Build') { - steps { - gitlabCommitStatus(name: 'Build') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Build', state: 'canceled' - echo "Milvus Build aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Build', state: 'failed' - echo "Milvus Build failure !" - } - } - } - } - - stage("Publish docker and helm") { - agent { - kubernetes { - label 'publish' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-docker - image: registry.zilliz.com/library/zilliz_docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -""" - } - } - stages { - stage('Publish Docker') { - steps { - gitlabCommitStatus(name: 'Publish Docker') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/nightly_publish_docker.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' - echo "Milvus Publish Docker aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' - echo "Milvus Publish Docker failure !" - } - } - } - } - - stage("Deploy to Development") { - parallel { - stage("Single Node") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: test-cluster-config -""" - } - } - - stages { - stage("Deploy to Dev") { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } - } - } - } - } - stage("Dev Test") { - steps { - gitlabCommitStatus(name: 'Deloy Test') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/dev_test_all.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" - } - } - } - } - } - stage ("Cleanup Dev") { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - } - } - } - post { - always { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - success { - script { - echo "Milvus Deploy to Dev Single Node CI/CD success !" - } - } - aborted { - script { - echo "Milvus Deploy to Dev Single Node CI/CD aborted !" - } - } - failure { - script { - echo "Milvus Deploy to Dev Single Node CI/CD failure !" - } - } - } - } - -// stage("Cluster") { -// agent { -// kubernetes { -// label 'dev-test' -// defaultContainer 'jnlp' -// yaml """ -// apiVersion: v1 -// kind: Pod -// metadata: -// labels: -// app: milvus -// componet: test -// spec: -// containers: -// - name: milvus-testframework -// image: registry.zilliz.com/milvus/milvus-test:v0.2 -// command: -// - cat -// tty: true -// volumeMounts: -// - name: kubeconf -// mountPath: /root/.kube/ -// readOnly: true -// volumes: -// - name: kubeconf -// secret: -// secretName: test-cluster-config -// """ -// } -// } -// stages { -// stage("Deploy to Dev") { -// steps { -// gitlabCommitStatus(name: 'Deloy to Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" -// } -// } -// } -// } -// } -// stage("Dev Test") { -// steps { -// gitlabCommitStatus(name: 'Deloy Test') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" -// load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" -// } -// } -// } -// } -// } -// stage ("Cleanup Dev") { -// steps { -// gitlabCommitStatus(name: 'Cleanup Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// } -// } -// } -// post { -// always { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// success { -// script { -// echo "Milvus Deploy to Dev Cluster CI/CD success !" -// } -// } -// aborted { -// script { -// echo "Milvus Deploy to Dev Cluster CI/CD aborted !" -// } -// } -// failure { -// script { -// echo "Milvus Deploy to Dev Cluster CI/CD failure !" -// } -// } -// } -// } - } - } - - stage("Deploy to Staging") { - parallel { - stage("Single Node") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: aks-gpu-cluster-config -""" - } - } - - stages { - stage("Deploy to Staging") { - steps { - gitlabCommitStatus(name: 'Deloy to Staging') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2staging.groovy" - } - } - } - } - } - stage("Staging Test") { - steps { - gitlabCommitStatus(name: 'Staging Test') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/staging_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_staging_test_out.groovy" - } - } - } - } - } - stage ("Cleanup Staging") { - steps { - gitlabCommitStatus(name: 'Cleanup Staging') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" - } - } - } - } - } - } - post { - always { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" - } - } - } - success { - script { - echo "Milvus Deploy to Staging Single Node CI/CD success !" - } - } - aborted { - script { - echo "Milvus Deploy to Staging Single Node CI/CD aborted !" - } - } - failure { - script { - echo "Milvus Deploy to Staging Single Node CI/CD failure !" - } - } - } - } - } - } - } - } - } - - post { - always { - script { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } - } - } - - success { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'success' - echo "Milvus CI/CD success !" - } - } - - aborted { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' - echo "Milvus CI/CD aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'failed' - echo "Milvus CI/CD failure !" - } - } - } -} - diff --git a/ci/pod_containers/milvus-engine-build.yaml b/ci/pod_containers/milvus-engine-build.yaml deleted file mode 100644 index cd5352ffef..0000000000 --- a/ci/pod_containers/milvus-engine-build.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: build-env -spec: - containers: - - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.9 - command: - - cat - tty: true diff --git a/ci/pod_containers/publish-docker.yaml b/ci/pod_containers/publish-docker.yaml deleted file mode 100644 index 268afb1331..0000000000 --- a/ci/pod_containers/publish-docker.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-docker - image: registry.zilliz.com/library/zilliz_docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock diff --git a/core/cmake/BuildUtils.cmake b/core/cmake/BuildUtils.cmake index 68cd22ae58..6332d29d74 100644 --- a/core/cmake/BuildUtils.cmake +++ b/core/cmake/BuildUtils.cmake @@ -74,7 +74,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path) ${CMAKE_COMMAND} -E echo "Extracting ${package_file} to ${install_path}" COMMAND - ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} + ${CMAKE_COMMAND} -E tar xzf ${package_file} ${install_path} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) diff --git a/core/src/cache/GpuCacheMgr.cpp b/core/src/cache/GpuCacheMgr.cpp index 72229527fa..1802fb3935 100644 --- a/core/src/cache/GpuCacheMgr.cpp +++ b/core/src/cache/GpuCacheMgr.cpp @@ -25,6 +25,7 @@ namespace milvus { namespace cache { +#ifdef MILVUS_GPU_VERSION std::mutex GpuCacheMgr::mutex_; std::unordered_map GpuCacheMgr::instance_; @@ -76,6 +77,7 @@ GpuCacheMgr::GetIndex(const std::string& key) { DataObjPtr obj = GetItem(key); return obj; } +#endif } // namespace cache } // namespace milvus diff --git a/core/src/cache/GpuCacheMgr.h b/core/src/cache/GpuCacheMgr.h index 4d434b2cfb..06dd44cca2 100644 --- a/core/src/cache/GpuCacheMgr.h +++ b/core/src/cache/GpuCacheMgr.h @@ -25,6 +25,7 @@ namespace milvus { namespace cache { +#ifdef MILVUS_GPU_VERSION class GpuCacheMgr; using GpuCacheMgrPtr = std::shared_ptr; @@ -42,6 +43,7 @@ class GpuCacheMgr : public CacheMgr { static std::mutex mutex_; static std::unordered_map instance_; }; +#endif } // namespace cache } // namespace milvus diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index dd230ce0d1..51ea665064 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -838,6 +838,25 @@ DBImpl::BackgroundBuildIndex() { // ENGINE_LOG_TRACE << "Background build index thread exit"; } +Status +DBImpl::GetFilesToBuildIndex(const std::string& table_id, const std::vector& file_types, + meta::TableFilesSchema& files) { + files.clear(); + auto status = meta_ptr_->FilesByType(table_id, file_types, files); + + // only build index for files that row count greater than certain threshold + for (auto it = files.begin(); it != files.end();) { + if ((*it).file_type_ == static_cast(meta::TableFileSchema::RAW) && + (*it).row_count_ < meta::BUILD_INDEX_THRESHOLD) { + it = files.erase(it); + } else { + it++; + } + } + + return Status::OK(); +} + Status DBImpl::GetFilesToSearch(const std::string& table_id, const std::vector& file_ids, const meta::DatesT& dates, meta::TableFilesSchema& files) { @@ -946,18 +965,18 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex } // get files to build index - std::vector file_ids; - auto status = meta_ptr_->FilesByType(table_id, file_types, file_ids); + meta::TableFilesSchema table_files; + auto status = GetFilesToBuildIndex(table_id, file_types, table_files); int times = 1; - while (!file_ids.empty()) { + while (!table_files.empty()) { ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; if (index.engine_type_ != (int)EngineType::FAISS_IDMAP) { status = meta_ptr_->UpdateTableFilesToIndex(table_id); } std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10 * 1000, times * 100))); - status = meta_ptr_->FilesByType(table_id, file_types, file_ids); + GetFilesToBuildIndex(table_id, file_types, table_files); times++; } diff --git a/core/src/db/DBImpl.h b/core/src/db/DBImpl.h index a0c5cc356d..bff56efded 100644 --- a/core/src/db/DBImpl.h +++ b/core/src/db/DBImpl.h @@ -152,6 +152,10 @@ class DBImpl : public DB { Status MemSerialize(); + Status + GetFilesToBuildIndex(const std::string& table_id, const std::vector& file_types, + meta::TableFilesSchema& files); + Status GetFilesToSearch(const std::string& table_id, const std::vector& file_ids, const meta::DatesT& dates, meta::TableFilesSchema& files); diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index ca307b90fc..c0ab4e829e 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -151,6 +151,7 @@ ExecutionEngineImpl::HybridLoad() const { return; } +#ifdef MILVUS_GPU_VERSION const std::string key = location_ + ".quantizer"; server::Config& config = server::Config::GetInstance(); @@ -205,6 +206,7 @@ ExecutionEngineImpl::HybridLoad() const { auto cache_quantizer = std::make_shared(quantizer); cache::GpuCacheMgr::GetInstance(best_device_id)->InsertItem(key, cache_quantizer); } +#endif } void @@ -342,6 +344,7 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { } #endif +#ifdef MILVUS_GPU_VERSION auto index = std::static_pointer_cast(cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_)); bool already_in_cache = (index != nullptr); if (already_in_cache) { @@ -364,16 +367,19 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { if (!already_in_cache) { GpuCache(device_id); } +#endif return Status::OK(); } Status ExecutionEngineImpl::CopyToIndexFileToGpu(uint64_t device_id) { +#ifdef MILVUS_GPU_VERSION gpu_num_ = device_id; auto to_index_data = std::make_shared(PhysicalSize()); cache::DataObjPtr obj = std::static_pointer_cast(to_index_data); milvus::cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(location_, obj); +#endif return Status::OK(); } @@ -584,15 +590,17 @@ ExecutionEngineImpl::Cache() { Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { +#ifdef MILVUS_GPU_VERSION cache::DataObjPtr obj = std::static_pointer_cast(index_); milvus::cache::GpuCacheMgr::GetInstance(gpu_id)->InsertItem(location_, obj); - +#endif return Status::OK(); } // TODO(linxj): remove. Status ExecutionEngineImpl::Init() { +#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); std::vector gpu_ids; Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids); @@ -604,6 +612,9 @@ ExecutionEngineImpl::Init() { std::string msg = "Invalid gpu_num"; return Status(SERVER_INVALID_ARGUMENT, msg); +#else + return Status::OK(); +#endif } } // namespace engine diff --git a/core/src/db/meta/Meta.h b/core/src/db/meta/Meta.h index f538bebce6..52fe86fe69 100644 --- a/core/src/db/meta/Meta.h +++ b/core/src/db/meta/Meta.h @@ -109,8 +109,7 @@ class Meta { FilesToIndex(TableFilesSchema&) = 0; virtual Status - FilesByType(const std::string& table_id, const std::vector& file_types, - std::vector& file_ids) = 0; + FilesByType(const std::string& table_id, const std::vector& file_types, TableFilesSchema& table_files) = 0; virtual Status Size(uint64_t& result) = 0; diff --git a/core/src/db/meta/MetaConsts.h b/core/src/db/meta/MetaConsts.h index 4e40ff7731..0c77dc2599 100644 --- a/core/src/db/meta/MetaConsts.h +++ b/core/src/db/meta/MetaConsts.h @@ -32,6 +32,13 @@ const size_t H_SEC = 60 * M_SEC; const size_t D_SEC = 24 * H_SEC; const size_t W_SEC = 7 * D_SEC; +// This value is to ignore small raw files when building index. +// The reason is: +// 1. The performance of brute-search for small raw files could be better than small index file. +// 2. And small raw files can be merged to larger files, thus reduce fragmented files count. +// We decide the value based on a testing for small size raw/index files. +const size_t BUILD_INDEX_THRESHOLD = 5000; + } // namespace meta } // namespace engine } // namespace milvus diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 4406b87f7e..6d13cad248 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -959,6 +959,7 @@ MySQLMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { updateTableFilesToIndexQuery << "UPDATE " << META_TABLEFILES << " SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " WHERE table_id = " << mysqlpp::quote << table_id + << " AND row_count >= " << std::to_string(meta::BUILD_INDEX_THRESHOLD) << " AND file_type = " << std::to_string(TableFileSchema::RAW) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFilesToIndex: " << updateTableFilesToIndexQuery.str(); @@ -1527,13 +1528,13 @@ MySQLMetaImpl::FilesToIndex(TableFilesSchema& files) { Status MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector& file_types, - std::vector& file_ids) { + TableFilesSchema& table_files) { if (file_types.empty()) { return Status(DB_ERROR, "file types array is empty"); } try { - file_ids.clear(); + table_files.clear(); mysqlpp::StoreQueryResult res; { @@ -1553,9 +1554,10 @@ MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector& mysqlpp::Query hasNonIndexFilesQuery = connectionPtr->query(); // since table_id is a unique column we just need to check whether it exists or not - hasNonIndexFilesQuery << "SELECT file_id, file_type" - << " FROM " << META_TABLEFILES << " WHERE table_id = " << mysqlpp::quote << table_id - << " AND file_type in (" << types << ");"; + hasNonIndexFilesQuery + << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on" + << " FROM " << META_TABLEFILES << " WHERE table_id = " << mysqlpp::quote << table_id + << " AND file_type in (" << types << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesByType: " << hasNonIndexFilesQuery.str(); @@ -1566,9 +1568,18 @@ MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector& int raw_count = 0, new_count = 0, new_merge_count = 0, new_index_count = 0; int to_index_count = 0, index_count = 0, backup_count = 0; for (auto& resRow : res) { - std::string file_id; - resRow["file_id"].to_string(file_id); - file_ids.push_back(file_id); + TableFileSchema file_schema; + file_schema.id_ = resRow["id"]; + file_schema.table_id_ = table_id; + file_schema.engine_type_ = resRow["engine_type"]; + resRow["file_id"].to_string(file_schema.file_id_); + file_schema.file_type_ = resRow["file_type"]; + file_schema.file_size_ = resRow["file_size"]; + file_schema.row_count_ = resRow["row_count"]; + file_schema.date_ = resRow["date"]; + file_schema.created_on_ = resRow["created_on"]; + + table_files.emplace_back(file_schema); int32_t file_type = resRow["file_type"]; switch (file_type) { diff --git a/core/src/db/meta/MySQLMetaImpl.h b/core/src/db/meta/MySQLMetaImpl.h index 00b7627548..dd882fca2e 100644 --- a/core/src/db/meta/MySQLMetaImpl.h +++ b/core/src/db/meta/MySQLMetaImpl.h @@ -108,7 +108,7 @@ class MySQLMetaImpl : public Meta { Status FilesByType(const std::string& table_id, const std::vector& file_types, - std::vector& file_ids) override; + TableFilesSchema& table_files) override; Status Archive() override; diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 12128c074d..74460c1b4d 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -58,7 +58,7 @@ HandleException(const std::string& desc, const char* what = nullptr) { } // namespace inline auto -StoragePrototype(const std::string &path) { +StoragePrototype(const std::string& path) { return make_storage(path, make_table(META_TABLES, make_column("id", &TableSchema::id_, primary_key()), @@ -160,7 +160,7 @@ SqliteMetaImpl::Initialize() { } Status -SqliteMetaImpl::CreateTable(TableSchema &table_schema) { +SqliteMetaImpl::CreateTable(TableSchema& table_schema) { try { server::MetricCollector metric; @@ -188,20 +188,20 @@ SqliteMetaImpl::CreateTable(TableSchema &table_schema) { try { auto id = ConnectorPtr->insert(table_schema); table_schema.id_ = id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when create table", e.what()); } ENGINE_LOG_DEBUG << "Successfully create table: " << table_schema.table_id_; return utils::CreateTablePath(options_, table_schema.table_id_); - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when create table", e.what()); } } Status -SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { +SqliteMetaImpl::DescribeTable(TableSchema& table_schema) { try { server::MetricCollector metric; @@ -218,7 +218,7 @@ SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { &TableSchema::partition_tag_, &TableSchema::version_), where(c(&TableSchema::table_id_) == table_schema.table_id_ - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); if (groups.size() == 1) { table_schema.id_ = std::get<0>(groups[0]); @@ -236,7 +236,7 @@ SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { } else { return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when describe table", e.what()); } @@ -244,20 +244,20 @@ SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { } Status -SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { +SqliteMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) { has_or_not = false; try { server::MetricCollector metric; auto tables = ConnectorPtr->select(columns(&TableSchema::id_), where(c(&TableSchema::table_id_) == table_id - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); if (tables.size() == 1) { has_or_not = true; } else { has_or_not = false; } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when lookup table", e.what()); } @@ -265,7 +265,7 @@ SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { } Status -SqliteMetaImpl::AllTables(std::vector &table_schema_array) { +SqliteMetaImpl::AllTables(std::vector& table_schema_array) { try { server::MetricCollector metric; @@ -281,8 +281,8 @@ SqliteMetaImpl::AllTables(std::vector &table_schema_array) { &TableSchema::owner_table_, &TableSchema::partition_tag_, &TableSchema::version_), - where(c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); - for (auto &table : selected) { + where(c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + for (auto& table : selected) { TableSchema schema; schema.id_ = std::get<0>(table); schema.table_id_ = std::get<1>(table); @@ -299,7 +299,7 @@ SqliteMetaImpl::AllTables(std::vector &table_schema_array) { table_schema_array.emplace_back(schema); } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when lookup all tables", e.what()); } @@ -307,7 +307,7 @@ SqliteMetaImpl::AllTables(std::vector &table_schema_array) { } Status -SqliteMetaImpl::DropTable(const std::string &table_id) { +SqliteMetaImpl::DropTable(const std::string& table_id) { try { server::MetricCollector metric; @@ -317,13 +317,13 @@ SqliteMetaImpl::DropTable(const std::string &table_id) { //soft delete table ConnectorPtr->update_all( set( - c(&TableSchema::state_) = (int) TableSchema::TO_DELETE), + c(&TableSchema::state_) = (int)TableSchema::TO_DELETE), where( c(&TableSchema::table_id_) == table_id and - c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); ENGINE_LOG_DEBUG << "Successfully delete table, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when delete table", e.what()); } @@ -331,7 +331,7 @@ SqliteMetaImpl::DropTable(const std::string &table_id) { } Status -SqliteMetaImpl::DeleteTableFiles(const std::string &table_id) { +SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) { try { server::MetricCollector metric; @@ -341,14 +341,14 @@ SqliteMetaImpl::DeleteTableFiles(const std::string &table_id) { //soft delete table files ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, + c(&TableFileSchema::file_type_) = (int)TableFileSchema::TO_DELETE, c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), where( c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); + c(&TableFileSchema::file_type_) != (int)TableFileSchema::TO_DELETE)); ENGINE_LOG_DEBUG << "Successfully delete table files, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when delete table files", e.what()); } @@ -356,7 +356,7 @@ SqliteMetaImpl::DeleteTableFiles(const std::string &table_id) { } Status -SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { +SqliteMetaImpl::CreateTableFile(TableFileSchema& file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = utils::GetDate(); } @@ -389,7 +389,7 @@ SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { ENGINE_LOG_DEBUG << "Successfully create table file, file id = " << file_schema.file_id_; return utils::CreateTableFilePath(options_, file_schema); - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when create table file", e.what()); } @@ -398,8 +398,8 @@ SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { // TODO(myh): Delete single vecotor by id Status -SqliteMetaImpl::DropDataByDate(const std::string &table_id, - const DatesT &dates) { +SqliteMetaImpl::DropDataByDate(const std::string& table_id, + const DatesT& dates) { if (dates.empty()) { return Status::OK(); } @@ -440,7 +440,7 @@ SqliteMetaImpl::DropDataByDate(const std::string &table_id, } ENGINE_LOG_DEBUG << "Successfully drop data by date, table id = " << table_schema.table_id_; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when drop partition", e.what()); } @@ -448,9 +448,9 @@ SqliteMetaImpl::DropDataByDate(const std::string &table_id, } Status -SqliteMetaImpl::GetTableFiles(const std::string &table_id, - const std::vector &ids, - TableFilesSchema &table_files) { +SqliteMetaImpl::GetTableFiles(const std::string& table_id, + const std::vector& ids, + TableFilesSchema& table_files) { try { table_files.clear(); auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, @@ -463,7 +463,7 @@ SqliteMetaImpl::GetTableFiles(const std::string &table_id, &TableFileSchema::created_on_), where(c(&TableFileSchema::table_id_) == table_id and in(&TableFileSchema::id_, ids) and - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); + c(&TableFileSchema::file_type_) != (int)TableFileSchema::TO_DELETE)); TableSchema table_schema; table_schema.table_id_ = table_id; auto status = DescribeTable(table_schema); @@ -472,7 +472,7 @@ SqliteMetaImpl::GetTableFiles(const std::string &table_id, } Status result; - for (auto &file : files) { + for (auto& file : files) { TableFileSchema file_schema; file_schema.table_id_ = table_id; file_schema.id_ = std::get<0>(file); @@ -495,13 +495,13 @@ SqliteMetaImpl::GetTableFiles(const std::string &table_id, ENGINE_LOG_DEBUG << "Get table files by id"; return result; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when lookup table files", e.what()); } } Status -SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { +SqliteMetaImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) { try { server::MetricCollector metric; @@ -512,7 +512,7 @@ SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { where( c(&TableSchema::table_id_) == table_id)); ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { std::string msg = "Encounter exception when update table flag: table_id = " + table_id; return HandleException(msg, e.what()); } @@ -521,7 +521,7 @@ SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { } Status -SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { +SqliteMetaImpl::UpdateTableFile(TableFileSchema& file_schema) { file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { server::MetricCollector metric; @@ -534,14 +534,14 @@ SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { //if the table has been deleted, just mark the table file as TO_DELETE //clean thread will delete the file later - if (tables.size() < 1 || std::get<0>(tables[0]) == (int) TableSchema::TO_DELETE) { + if (tables.size() < 1 || std::get<0>(tables[0]) == (int)TableSchema::TO_DELETE) { file_schema.file_type_ = TableFileSchema::TO_DELETE; } ConnectorPtr->update(file_schema); ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; - } catch (std::exception &e) { + } catch (std::exception& e) { std::string msg = "Exception update table file: table_id = " + file_schema.table_id_ + " file_id = " + file_schema.file_id_; return HandleException(msg, e.what()); @@ -550,7 +550,7 @@ SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { } Status -SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { +SqliteMetaImpl::UpdateTableFiles(TableFilesSchema& files) { try { server::MetricCollector metric; @@ -558,13 +558,13 @@ SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { std::lock_guard meta_lock(meta_mutex_); std::map has_tables; - for (auto &file : files) { + for (auto& file : files) { if (has_tables.find(file.table_id_) != has_tables.end()) { continue; } auto tables = ConnectorPtr->select(columns(&TableSchema::id_), where(c(&TableSchema::table_id_) == file.table_id_ - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); if (tables.size() >= 1) { has_tables[file.table_id_] = true; } else { @@ -573,7 +573,7 @@ SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { } auto commited = ConnectorPtr->transaction([&]() mutable { - for (auto &file : files) { + for (auto& file : files) { if (!has_tables[file.table_id_]) { file.file_type_ = TableFileSchema::TO_DELETE; } @@ -589,7 +589,7 @@ SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { } ENGINE_LOG_DEBUG << "Update " << files.size() << " table files"; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when update table files", e.what()); } return Status::OK(); @@ -613,7 +613,7 @@ SqliteMetaImpl::UpdateTableIndex(const std::string& table_id, const TableIndex& &TableSchema::partition_tag_, &TableSchema::version_), where(c(&TableSchema::table_id_) == table_id - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); if (tables.size() > 0) { meta::TableSchema table_schema; @@ -639,11 +639,11 @@ SqliteMetaImpl::UpdateTableIndex(const std::string& table_id, const TableIndex& //set all backup file to raw ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, + c(&TableFileSchema::file_type_) = (int)TableFileSchema::RAW, c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), where( c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP)); + c(&TableFileSchema::file_type_) == (int)TableFileSchema::BACKUP)); ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; } catch (std::exception& e) { @@ -655,7 +655,7 @@ SqliteMetaImpl::UpdateTableIndex(const std::string& table_id, const TableIndex& } Status -SqliteMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { +SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { try { server::MetricCollector metric; @@ -664,13 +664,14 @@ SqliteMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_INDEX), + c(&TableFileSchema::file_type_) = (int)TableFileSchema::TO_INDEX), where( c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW)); + c(&TableFileSchema::row_count_) >= meta::BUILD_INDEX_THRESHOLD and + c(&TableFileSchema::file_type_) == (int)TableFileSchema::RAW)); ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when update table files to to_index", e.what()); } @@ -686,7 +687,7 @@ SqliteMetaImpl::DescribeTableIndex(const std::string& table_id, TableIndex& inde &TableSchema::nlist_, &TableSchema::metric_type_), where(c(&TableSchema::table_id_) == table_id - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); if (groups.size() == 1) { index.engine_type_ = std::get<0>(groups[0]); @@ -713,20 +714,20 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) { //soft delete index files ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, + c(&TableFileSchema::file_type_) = (int)TableFileSchema::TO_DELETE, c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), where( c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX)); + c(&TableFileSchema::file_type_) == (int)TableFileSchema::INDEX)); //set all backup file to raw ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, + c(&TableFileSchema::file_type_) = (int)TableFileSchema::RAW, c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), where( c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP)); + c(&TableFileSchema::file_type_) == (int)TableFileSchema::BACKUP)); //set table index type to raw ConnectorPtr->update_all( @@ -738,7 +739,7 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) { c(&TableSchema::table_id_) == table_id)); ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when delete table index files", e.what()); } @@ -746,7 +747,9 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) { } Status -SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag) { +SqliteMetaImpl::CreatePartition(const std::string& table_id, + const std::string& partition_name, + const std::string& tag) { server::MetricCollector metric; TableSchema table_schema; @@ -757,7 +760,7 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& } // not allow create partition under partition - if(!table_schema.owner_table_.empty()) { + if (!table_schema.owner_table_.empty()) { return Status(DB_ERROR, "Nested partition is not allowed"); } @@ -769,7 +772,7 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& // not allow duplicated partition std::string exist_partition; GetPartitionName(table_id, valid_tag, exist_partition); - if(!exist_partition.empty()) { + if (!exist_partition.empty()) { return Status(DB_ERROR, "Duplicate partition is not allowed"); } @@ -805,16 +808,16 @@ SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vectorselect(columns(&TableSchema::table_id_), - where(c(&TableSchema::owner_table_) == table_id - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); - for(size_t i = 0; i < partitions.size(); i++) { + where(c(&TableSchema::owner_table_) == table_id + and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + for (size_t i = 0; i < partitions.size(); i++) { std::string partition_name = std::get<0>(partitions[i]); meta::TableSchema partition_schema; partition_schema.table_id_ = partition_name; DescribeTable(partition_schema); partiton_schema_array.emplace_back(partition_schema); } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when show partitions", e.what()); } @@ -832,14 +835,15 @@ SqliteMetaImpl::GetPartitionName(const std::string& table_id, const std::string& server::StringHelpFunctions::TrimStringBlank(valid_tag); auto name = ConnectorPtr->select(columns(&TableSchema::table_id_), - where(c(&TableSchema::owner_table_) == table_id - and c(&TableSchema::partition_tag_) == valid_tag)); + where(c(&TableSchema::owner_table_) == table_id + and c(&TableSchema::partition_tag_) == valid_tag + and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); if (name.size() > 0) { partition_name = std::get<0>(name[0]); } else { return Status(DB_NOT_FOUND, "Table " + table_id + "'s partition " + valid_tag + " not found"); } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when get partition name", e.what()); } @@ -1032,7 +1036,7 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile } Status -SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { +SqliteMetaImpl::FilesToIndex(TableFilesSchema& files) { files.clear(); try { @@ -1048,13 +1052,13 @@ SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { &TableFileSchema::engine_type_, &TableFileSchema::created_on_), where(c(&TableFileSchema::file_type_) - == (int) TableFileSchema::TO_INDEX)); + == (int)TableFileSchema::TO_INDEX)); std::map groups; TableFileSchema table_file; Status ret; - for (auto &file : selected) { + for (auto& file : selected) { table_file.id_ = std::get<0>(file); table_file.table_id_ = std::get<1>(file); table_file.file_id_ = std::get<2>(file); @@ -1090,48 +1094,66 @@ SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-index files"; } return ret; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when iterate raw files", e.what()); } } Status -SqliteMetaImpl::FilesByType(const std::string &table_id, - const std::vector &file_types, - std::vector &file_ids) { +SqliteMetaImpl::FilesByType(const std::string& table_id, + const std::vector& file_types, + TableFilesSchema& table_files) { if (file_types.empty()) { return Status(DB_ERROR, "file types array is empty"); } try { - file_ids.clear(); - auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_, - &TableFileSchema::file_type_), + table_files.clear(); + auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::file_id_, + &TableFileSchema::file_type_, + &TableFileSchema::file_size_, + &TableFileSchema::row_count_, + &TableFileSchema::date_, + &TableFileSchema::engine_type_, + &TableFileSchema::created_on_), where(in(&TableFileSchema::file_type_, file_types) and c(&TableFileSchema::table_id_) == table_id)); if (selected.size() >= 1) { int raw_count = 0, new_count = 0, new_merge_count = 0, new_index_count = 0; int to_index_count = 0, index_count = 0, backup_count = 0; - for (auto &file : selected) { - file_ids.push_back(std::get<0>(file)); - switch (std::get<1>(file)) { - case (int) TableFileSchema::RAW:raw_count++; + for (auto& file : selected) { + TableFileSchema file_schema; + file_schema.table_id_ = table_id; + file_schema.id_ = std::get<0>(file); + file_schema.file_id_ = std::get<1>(file); + file_schema.file_type_ = std::get<2>(file); + file_schema.file_size_ = std::get<3>(file); + file_schema.row_count_ = std::get<4>(file); + file_schema.date_ = std::get<5>(file); + file_schema.engine_type_ = std::get<6>(file); + file_schema.created_on_ = std::get<7>(file); + + switch (file_schema.file_type_) { + case (int)TableFileSchema::RAW:raw_count++; break; - case (int) TableFileSchema::NEW:new_count++; + case (int)TableFileSchema::NEW:new_count++; break; - case (int) TableFileSchema::NEW_MERGE:new_merge_count++; + case (int)TableFileSchema::NEW_MERGE:new_merge_count++; break; - case (int) TableFileSchema::NEW_INDEX:new_index_count++; + case (int)TableFileSchema::NEW_INDEX:new_index_count++; break; - case (int) TableFileSchema::TO_INDEX:to_index_count++; + case (int)TableFileSchema::TO_INDEX:to_index_count++; break; - case (int) TableFileSchema::INDEX:index_count++; + case (int)TableFileSchema::INDEX:index_count++; break; - case (int) TableFileSchema::BACKUP:backup_count++; + case (int)TableFileSchema::BACKUP:backup_count++; break; default:break; } + + table_files.emplace_back(file_schema); } ENGINE_LOG_DEBUG << "Table " << table_id << " currently has raw files:" << raw_count @@ -1139,13 +1161,12 @@ SqliteMetaImpl::FilesByType(const std::string &table_id, << " new_index files:" << new_index_count << " to_index files:" << to_index_count << " index files:" << index_count << " backup files:" << backup_count; } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when check non index files", e.what()); } return Status::OK(); } - // TODO(myh): Support swap to cloud storage Status SqliteMetaImpl::Archive() { @@ -1166,11 +1187,11 @@ SqliteMetaImpl::Archive() { ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE), + c(&TableFileSchema::file_type_) = (int)TableFileSchema::TO_DELETE), where( - c(&TableFileSchema::created_on_) < (int64_t) (now - usecs) and - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); - } catch (std::exception &e) { + c(&TableFileSchema::created_on_) < (int64_t)(now - usecs) and + c(&TableFileSchema::file_type_) != (int)TableFileSchema::TO_DELETE)); + } catch (std::exception& e) { return HandleException("Encounter exception when update table files", e.what()); } @@ -1218,15 +1239,15 @@ SqliteMetaImpl::CleanUp() { std::lock_guard meta_lock(meta_mutex_); std::vector file_types = { - (int) TableFileSchema::NEW, - (int) TableFileSchema::NEW_INDEX, - (int) TableFileSchema::NEW_MERGE + (int)TableFileSchema::NEW, + (int)TableFileSchema::NEW_INDEX, + (int)TableFileSchema::NEW_MERGE }; auto files = ConnectorPtr->select(columns(&TableFileSchema::id_), where(in(&TableFileSchema::file_type_, file_types))); auto commited = ConnectorPtr->transaction([&]() mutable { - for (auto &file : files) { + for (auto& file : files) { ENGINE_LOG_DEBUG << "Remove table file type as NEW"; ConnectorPtr->remove(std::get<0>(file)); } @@ -1240,7 +1261,7 @@ SqliteMetaImpl::CleanUp() { if (files.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << files.size() << " files"; } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when clean table file", e.what()); } @@ -1265,7 +1286,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { &TableFileSchema::date_), where( c(&TableFileSchema::file_type_) == - (int) TableFileSchema::TO_DELETE + (int)TableFileSchema::TO_DELETE and c(&TableFileSchema::updated_time_) < now - seconds * US_PS)); @@ -1354,7 +1375,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } Status -SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { +SqliteMetaImpl::Count(const std::string& table_id, uint64_t& result) { try { server::MetricCollector metric; @@ -1414,14 +1435,14 @@ SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, &TableFileSchema::file_size_), where(c(&TableFileSchema::file_type_) - != (int) TableFileSchema::TO_DELETE), + != (int)TableFileSchema::TO_DELETE), order_by(&TableFileSchema::id_), limit(10)); std::vector ids; TableFileSchema table_file; - for (auto &file : selected) { + for (auto& file : selected) { if (to_discard_size <= 0) break; table_file.id_ = std::get<0>(file); table_file.file_size_ = std::get<1>(file); @@ -1437,7 +1458,7 @@ SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) { ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, + c(&TableFileSchema::file_type_) = (int)TableFileSchema::TO_DELETE, c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), where( in(&TableFileSchema::id_, ids))); @@ -1448,7 +1469,7 @@ SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) { if (!commited) { return HandleException("DiscardFiles error: sqlite transaction failed"); } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("Encounter exception when discard table file", e.what()); } diff --git a/core/src/db/meta/SqliteMetaImpl.h b/core/src/db/meta/SqliteMetaImpl.h index 84d97ed49d..8e821d81de 100644 --- a/core/src/db/meta/SqliteMetaImpl.h +++ b/core/src/db/meta/SqliteMetaImpl.h @@ -108,7 +108,7 @@ class SqliteMetaImpl : public Meta { Status FilesByType(const std::string& table_id, const std::vector& file_types, - std::vector& file_ids) override; + TableFilesSchema& table_files) override; Status Size(uint64_t& result) override; diff --git a/core/src/main.cpp b/core/src/main.cpp index b39ba87997..5c97a061d2 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -59,9 +59,9 @@ print_banner() { #endif << " library." << std::endl; #ifdef MILVUS_CPU_VERSION - std::cout << "You are using Milvus CPU version" << std::endl; + std::cout << "You are using Milvus CPU edition" << std::endl; #else - std::cout << "You are using Milvus GPU version" << std::endl; + std::cout << "You are using Milvus GPU edition" << std::endl; #endif std::cout << std::endl; } diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 69d293f986..f86b6f44b3 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -54,36 +54,40 @@ load_simple_config() { // get resources #ifdef MILVUS_GPU_VERSION + bool enable_gpu = false; server::Config& config = server::Config::GetInstance(); - std::vector gpu_ids; - config.GetGpuResourceConfigSearchResources(gpu_ids); - std::vector build_gpu_ids; - config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids); - auto pcie = Connection("pcie", 12000); + config.GetGpuResourceConfigEnable(enable_gpu); + if (enable_gpu) { + std::vector gpu_ids; + config.GetGpuResourceConfigSearchResources(gpu_ids); + std::vector build_gpu_ids; + config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids); + auto pcie = Connection("pcie", 12000); - std::vector not_find_build_ids; - for (auto& build_id : build_gpu_ids) { - bool find_gpu_id = false; - for (auto& gpu_id : gpu_ids) { - if (gpu_id == build_id) { - find_gpu_id = true; - break; + std::vector not_find_build_ids; + for (auto& build_id : build_gpu_ids) { + bool find_gpu_id = false; + for (auto& gpu_id : gpu_ids) { + if (gpu_id == build_id) { + find_gpu_id = true; + break; + } + } + if (not find_gpu_id) { + not_find_build_ids.emplace_back(build_id); } } - if (not find_gpu_id) { - not_find_build_ids.emplace_back(build_id); + + for (auto& gpu_id : gpu_ids) { + ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); } - } - for (auto& gpu_id : gpu_ids) { - ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); - } - - for (auto& not_find_id : not_find_build_ids) { - ResMgrInst::GetInstance()->Add( - ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie); + for (auto& not_find_id : not_find_build_ids) { + ResMgrInst::GetInstance()->Add( + ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie); + } } #endif } diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index dc2d5ade35..1e8a7acf2e 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -102,11 +102,35 @@ class OptimizerInst { if (instance == nullptr) { std::vector pass_list; #ifdef MILVUS_GPU_VERSION - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); + bool enable_gpu = false; + server::Config& config = server::Config::GetInstance(); + config.GetGpuResourceConfigEnable(enable_gpu); + if (enable_gpu) { + std::vector build_gpus; + std::vector search_gpus; + int64_t gpu_search_threshold; + config.GetGpuResourceConfigBuildIndexResources(build_gpus); + config.GetGpuResourceConfigSearchResources(search_gpus); + config.GetEngineConfigGpuSearchThreshold(gpu_search_threshold); + std::string build_msg = "Build index gpu:"; + for (auto build_id : build_gpus) { + build_msg.append(" gpu" + std::to_string(build_id)); + } + SERVER_LOG_DEBUG << build_msg; + + std::string search_msg = "Search gpu:"; + for (auto search_id : search_gpus) { + search_msg.append(" gpu" + std::to_string(search_id)); + } + search_msg.append(". gpu_search_threshold:" + std::to_string(gpu_search_threshold)); + SERVER_LOG_DEBUG << search_msg; + + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + } #endif pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index d535b9675f..67036f9d7b 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -25,12 +25,13 @@ namespace scheduler { void BuildIndexPass::Init() { +#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); - std::vector build_resources; - Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); + Status s = config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids_); if (!s.ok()) { throw; } +#endif } bool @@ -38,13 +39,16 @@ BuildIndexPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::BuildIndexTask) return false; - if (build_gpu_ids_.empty()) + if (build_gpu_ids_.empty()) { + SERVER_LOG_WARNING << "BuildIndexPass cannot get build index gpu!"; return false; + } ResourcePtr res_ptr; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_gpu_ids_[specified_gpu_id_]); auto label = std::make_shared(std::weak_ptr(res_ptr)); task->label() = label; + SERVER_LOG_DEBUG << "Specify gpu" << specified_gpu_id_ << " to build index!"; specified_gpu_id_ = (specified_gpu_id_ + 1) % build_gpu_ids_.size(); return true; diff --git a/core/src/scheduler/optimizer/BuildIndexPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h index 4f7117fc4e..3adf1259a7 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.h +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -45,7 +45,7 @@ class BuildIndexPass : public Pass { private: uint64_t specified_gpu_id_ = 0; - std::vector build_gpu_ids_; + std::vector build_gpu_ids_; }; using BuildIndexPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissFlatPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp index 61ca1b9ec9..53256f7790 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -29,6 +29,7 @@ namespace scheduler { void FaissFlatPass::Init() { +#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { @@ -38,6 +39,7 @@ FaissFlatPass::Init() { if (!s.ok()) { throw; } +#endif } bool @@ -54,9 +56,11 @@ FaissFlatPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (search_job->nq() < threshold_) { + SERVER_LOG_DEBUG << "FaissFlatPass: nq < gpu_search_threshold, specify cpu to search!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % gpus.size(); + SERVER_LOG_DEBUG << "FaissFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!"; count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp index 1f1efb374b..8ad51fb14a 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -29,6 +29,7 @@ namespace scheduler { void FaissIVFFlatPass::Init() { +#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { @@ -38,6 +39,7 @@ FaissIVFFlatPass::Init() { if (!s.ok()) { throw; } +#endif } bool @@ -54,9 +56,12 @@ FaissIVFFlatPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (search_job->nq() < threshold_) { + SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq < gpu_search_threshold, specify cpu to search!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % gpus.size(); + SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id + << " to search!"; count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index a99e861e03..cc04944fda 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -29,12 +29,14 @@ namespace scheduler { void FaissIVFSQ8HPass::Init() { +#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { threshold_ = std::numeric_limits::max(); } s = config.GetGpuResourceConfigSearchResources(gpus); +#endif } bool @@ -51,9 +53,12 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (search_job->nq() < threshold_) { + SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq < gpu_search_threshold, specify cpu to search!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % gpus.size(); + SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq > gpu_search_threshold, specify gpu" << best_device_id + << " to search!"; count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp index 30dd306b3b..c2a1858966 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -29,6 +29,7 @@ namespace scheduler { void FaissIVFSQ8Pass::Init() { +#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { @@ -38,6 +39,7 @@ FaissIVFSQ8Pass::Init() { if (!s.ok()) { throw; } +#endif } bool @@ -54,9 +56,12 @@ FaissIVFSQ8Pass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); ResourcePtr res_ptr; if (search_job->nq() < threshold_) { + SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq < gpu_search_threshold, specify cpu to search!"; res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); } else { auto best_device_id = count_ % gpus.size(); + SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq > gpu_search_threshold, specify gpu" << best_device_id + << " to search!"; count_++; res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } diff --git a/core/src/scheduler/optimizer/FallbackPass.cpp b/core/src/scheduler/optimizer/FallbackPass.cpp index 2e275ede4b..15687bb9bb 100644 --- a/core/src/scheduler/optimizer/FallbackPass.cpp +++ b/core/src/scheduler/optimizer/FallbackPass.cpp @@ -33,6 +33,7 @@ FallbackPass::Run(const TaskPtr& task) { return false; } // NEVER be empty + SERVER_LOG_DEBUG << "FallbackPass!"; auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0]; auto label = std::make_shared(cpu); task->label() = label; diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index f561fa947d..e952bd0938 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -85,7 +85,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) { size_t file_size = to_index_engine_->PhysicalSize(); - std::string info = "Load file id:" + std::to_string(file_->id_) + + std::string info = "Load file id:" + std::to_string(file_->id_) + " " + type_str + " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; double span = rc.ElapseFromBegin(info); diff --git a/core/src/sdk/examples/utils/Utils.cpp b/core/src/sdk/examples/utils/Utils.cpp index fa373cd498..d3bf9eec25 100644 --- a/core/src/sdk/examples/utils/Utils.cpp +++ b/core/src/sdk/examples/utils/Utils.cpp @@ -99,6 +99,12 @@ Utils::IndexTypeName(const milvus::IndexType& index_type) { return "NSG"; case milvus::IndexType::IVFSQ8H: return "IVFSQ8H"; + case milvus::IndexType::IVFPQ: + return "IVFPQ"; + case milvus::IndexType::SPTAGKDT: + return "SPTAGKDT"; + case milvus::IndexType::SPTAGBKT: + return "SPTAGBKT"; default: return "Unknown index type"; } diff --git a/core/src/sdk/include/MilvusApi.h b/core/src/sdk/include/MilvusApi.h index 9fa98deb40..5c7736d4e2 100644 --- a/core/src/sdk/include/MilvusApi.h +++ b/core/src/sdk/include/MilvusApi.h @@ -37,6 +37,9 @@ enum class IndexType { IVFSQ8 = 3, NSG = 4, IVFSQ8H = 5, + IVFPQ = 6, + SPTAGKDT = 7, + SPTAGBKT = 8, }; enum class MetricType { diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index f3efcff0cc..95bab84391 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -182,6 +182,7 @@ Config::ValidateConfig() { return s; } +#ifdef MILVUS_GPU_VERSION int64_t engine_gpu_search_threshold; s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold); if (!s.ok()) { @@ -189,35 +190,36 @@ Config::ValidateConfig() { } /* gpu resource config */ -#ifdef MILVUS_GPU_VERSION bool gpu_resource_enable; s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - int64_t resource_cache_capacity; - s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity); - if (!s.ok()) { - return s; - } + if (gpu_resource_enable) { + int64_t resource_cache_capacity; + s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity); + if (!s.ok()) { + return s; + } - float resource_cache_threshold; - s = GetGpuResourceConfigCacheThreshold(resource_cache_threshold); - if (!s.ok()) { - return s; - } + float resource_cache_threshold; + s = GetGpuResourceConfigCacheThreshold(resource_cache_threshold); + if (!s.ok()) { + return s; + } - std::vector search_resources; - s = GetGpuResourceConfigSearchResources(search_resources); - if (!s.ok()) { - return s; - } + std::vector search_resources; + s = GetGpuResourceConfigSearchResources(search_resources); + if (!s.ok()) { + return s; + } - std::vector index_build_resources; - s = GetGpuResourceConfigBuildIndexResources(index_build_resources); - if (!s.ok()) { - return s; + std::vector index_build_resources; + s = GetGpuResourceConfigBuildIndexResources(index_build_resources); + if (!s.ok()) { + return s; + } } #endif @@ -323,13 +325,13 @@ Config::ResetDefaultConfig() { return s; } +#ifdef MILVUS_GPU_VERSION + /* gpu resource config */ s = SetEngineConfigGpuSearchThreshold(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); if (!s.ok()) { return s; } - /* gpu resource config */ -#ifdef MILVUS_GPU_VERSION s = SetGpuResourceConfigEnable(CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); if (!s.ok()) { return s; @@ -630,6 +632,7 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { return Status::OK(); } +#ifdef MILVUS_GPU_VERSION Status Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { @@ -759,6 +762,7 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector return Status::OK(); } +#endif //////////////////////////////////////////////////////////////////////////////// ConfigNode& @@ -979,6 +983,7 @@ Config::GetEngineConfigOmpThreadNum(int64_t& value) { return Status::OK(); } +#ifdef MILVUS_GPU_VERSION Status Config::GetEngineConfigGpuSearchThreshold(int64_t& value) { std::string str = @@ -1095,6 +1100,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { } return Status::OK(); } +#endif /////////////////////////////////////////////////////////////////////////////// /* server config */ @@ -1282,6 +1288,8 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { return Status::OK(); } +#ifdef MILVUS_GPU_VERSION +/* gpu resource config */ Status Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { Status s = CheckEngineConfigGpuSearchThreshold(value); @@ -1292,7 +1300,6 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { return Status::OK(); } -/* gpu resource config */ Status Config::SetGpuResourceConfigEnable(const std::string& value) { Status s = CheckGpuResourceConfigEnable(value); @@ -1346,6 +1353,7 @@ Config::SetGpuResourceConfigBuildIndexResources(const std::string& value) { SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, value); return Status::OK(); } // namespace server +#endif } // namespace server } // namespace milvus diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 0907080a6f..281a832d57 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -170,6 +170,8 @@ class Config { CheckEngineConfigUseBlasThreshold(const std::string& value); Status CheckEngineConfigOmpThreadNum(const std::string& value); + +#ifdef MILVUS_GPU_VERSION Status CheckEngineConfigGpuSearchThreshold(const std::string& value); @@ -184,6 +186,7 @@ class Config { CheckGpuResourceConfigSearchResources(const std::vector& value); Status CheckGpuResourceConfigBuildIndexResources(const std::vector& value); +#endif std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -239,6 +242,8 @@ class Config { GetEngineConfigUseBlasThreshold(int64_t& value); Status GetEngineConfigOmpThreadNum(int64_t& value); + +#ifdef MILVUS_GPU_VERSION Status GetEngineConfigGpuSearchThreshold(int64_t& value); @@ -253,6 +258,7 @@ class Config { GetGpuResourceConfigSearchResources(std::vector& value); Status GetGpuResourceConfigBuildIndexResources(std::vector& value); +#endif public: /* server config */ @@ -300,6 +306,8 @@ class Config { SetEngineConfigUseBlasThreshold(const std::string& value); Status SetEngineConfigOmpThreadNum(const std::string& value); + +#ifdef MILVUS_GPU_VERSION Status SetEngineConfigGpuSearchThreshold(const std::string& value); @@ -314,6 +322,7 @@ class Config { SetGpuResourceConfigSearchResources(const std::string& value); Status SetGpuResourceConfigBuildIndexResources(const std::string& value); +#endif private: std::unordered_map> config_map_; diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index 5676504722..169463080e 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -183,7 +183,11 @@ Server::Start() { // print version information SERVER_LOG_INFO << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME; - +#ifdef MILVUS_CPU_VERSION + SERVER_LOG_INFO << "CPU edition"; +#else + SERVER_LOG_INFO << "GPU edition"; +#endif server::Metrics::GetInstance().Init(); server::SystemInfo::GetInstance().Init(); diff --git a/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp b/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp index e1eb07af5c..0f46217057 100644 --- a/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp +++ b/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp @@ -90,8 +90,8 @@ GrpcBaseRequest::SetStatus(ErrorCode error_code, const std::string& error_msg) { std::string GrpcBaseRequest::TableNotExistMsg(const std::string& table_name) { return "Table " + table_name + - " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name " - "exists."; + " does not exist. Use milvus.has_table to verify whether the table exists. " + "You also can check whether the table name exists."; } Status diff --git a/core/src/utils/StringHelpFunctions.h b/core/src/utils/StringHelpFunctions.h index 3a41e53f4b..51812fae40 100644 --- a/core/src/utils/StringHelpFunctions.h +++ b/core/src/utils/StringHelpFunctions.h @@ -30,9 +30,13 @@ class StringHelpFunctions { StringHelpFunctions() = default; public: + // trim blanks from begin and end + // " a b c " => "a b c" static void TrimStringBlank(std::string& string); + // trim quotes from begin and end + // "'abc'" => "abc" static void TrimStringQuote(std::string& string, const std::string& qoute); @@ -46,6 +50,8 @@ class StringHelpFunctions { static void SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector& result); + // merge strings with delimeter + // "a", "b", "c" => "a,b,c" static void MergeStringWithDelimeter(const std::vector& strs, const std::string& delimeter, std::string& result); diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index 12b2372fc5..2d1a0e257e 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -218,10 +218,9 @@ ValidationUtil::ValidateGpuIndex(int32_t gpu_index) { return Status::OK(); } +#ifdef MILVUS_GPU_VERSION Status ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) { -#ifdef MILVUS_GPU_VERSION - cudaDeviceProp deviceProp; auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); if (cuda_err) { @@ -232,10 +231,9 @@ ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) { } memory = deviceProp.totalGlobalMem; -#endif - return Status::OK(); } +#endif Status ValidationUtil::ValidateIpAddress(const std::string& ip_address) { diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index ab32c35c40..bc523654e5 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -64,8 +64,10 @@ class ValidationUtil { static Status ValidateGpuIndex(int32_t gpu_index); +#ifdef MILVUS_GPU_VERSION static Status GetGpuMemory(int32_t gpu_index, size_t& memory); +#endif static Status ValidateIpAddress(const std::string& ip_address); diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index 5a2296b16e..42105777aa 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -37,6 +37,16 @@ constexpr int64_t M_BYTE = 1024 * 1024; Status KnowhereResource::Initialize() { #ifdef MILVUS_GPU_VERSION + Status s; + bool enable_gpu = false; + server::Config& config = server::Config::GetInstance(); + s = config.GetGpuResourceConfigEnable(enable_gpu); + if (!s.ok()) + return s; + + if (not enable_gpu) + return Status::OK(); + struct GpuResourceSetting { int64_t pinned_memory = 300 * M_BYTE; int64_t temp_memory = 300 * M_BYTE; @@ -44,10 +54,8 @@ KnowhereResource::Initialize() { }; using GpuResourcesArray = std::map; GpuResourcesArray gpu_resources; - Status s; // get build index gpu resource - server::Config& config = server::Config::GetInstance(); std::vector build_index_gpus; s = config.GetGpuResourceConfigBuildIndexResources(build_index_gpus); if (!s.ok()) diff --git a/core/unittest/db/test_meta.cpp b/core/unittest/db/test_meta.cpp index 097f004bd1..143bf39383 100644 --- a/core/unittest/db/test_meta.cpp +++ b/core/unittest/db/test_meta.cpp @@ -306,9 +306,9 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_EQ(dated_files[table_file.date_].size(), 0); std::vector file_types; - std::vector file_ids; - status = impl_->FilesByType(table.table_id_, file_types, file_ids); - ASSERT_TRUE(file_ids.empty()); + milvus::engine::meta::TableFilesSchema table_files; + status = impl_->FilesByType(table.table_id_, file_types, table_files); + ASSERT_TRUE(table_files.empty()); ASSERT_FALSE(status.ok()); file_types = { @@ -317,11 +317,11 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { milvus::engine::meta::TableFileSchema::INDEX, milvus::engine::meta::TableFileSchema::RAW, milvus::engine::meta::TableFileSchema::BACKUP, }; - status = impl_->FilesByType(table.table_id_, file_types, file_ids); + status = impl_->FilesByType(table.table_id_, file_types, table_files); ASSERT_TRUE(status.ok()); uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt + backup_files_cnt + new_files_cnt + raw_files_cnt + to_index_files_cnt + index_files_cnt; - ASSERT_EQ(file_ids.size(), total_cnt); + ASSERT_EQ(table_files.size(), total_cnt); status = impl_->DeleteTableFiles(table_id); ASSERT_TRUE(status.ok()); diff --git a/core/unittest/db/test_meta_mysql.cpp b/core/unittest/db/test_meta_mysql.cpp index b9a82c0748..9a52a01b7b 100644 --- a/core/unittest/db/test_meta_mysql.cpp +++ b/core/unittest/db/test_meta_mysql.cpp @@ -169,9 +169,9 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { std::vector file_types = { (int)milvus::engine::meta::TableFileSchema::NEW, }; - std::vector file_ids; - status = impl.FilesByType(table_id, file_types, file_ids); - ASSERT_FALSE(file_ids.empty()); + milvus::engine::meta::TableFilesSchema table_files; + status = impl.FilesByType(table_id, file_types, table_files); + ASSERT_FALSE(table_files.empty()); status = impl.UpdateTableFilesToIndex(table_id); ASSERT_TRUE(status.ok()); @@ -326,9 +326,9 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { ASSERT_EQ(dated_files[table_file.date_].size(), 0); std::vector file_types; - std::vector file_ids; - status = impl_->FilesByType(table.table_id_, file_types, file_ids); - ASSERT_TRUE(file_ids.empty()); + milvus::engine::meta::TableFilesSchema table_files; + status = impl_->FilesByType(table.table_id_, file_types, table_files); + ASSERT_TRUE(table_files.empty()); ASSERT_FALSE(status.ok()); file_types = { @@ -337,11 +337,11 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { milvus::engine::meta::TableFileSchema::INDEX, milvus::engine::meta::TableFileSchema::RAW, milvus::engine::meta::TableFileSchema::BACKUP, }; - status = impl_->FilesByType(table.table_id_, file_types, file_ids); + status = impl_->FilesByType(table.table_id_, file_types, table_files); ASSERT_TRUE(status.ok()); uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt + backup_files_cnt + new_files_cnt + raw_files_cnt + to_index_files_cnt + index_files_cnt; - ASSERT_EQ(file_ids.size(), total_cnt); + ASSERT_EQ(table_files.size(), total_cnt); status = impl_->DeleteTableFiles(table_id); ASSERT_TRUE(status.ok()); diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index afa1d39006..293eeccc69 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -132,8 +132,8 @@ BaseTest::SetUp() { void BaseTest::TearDown() { milvus::cache::CpuCacheMgr::GetInstance()->ClearCache(); - milvus::cache::GpuCacheMgr::GetInstance(0)->ClearCache(); #ifdef MILVUS_GPU_VERSION + milvus::cache::GpuCacheMgr::GetInstance(0)->ClearCache(); knowhere::FaissGpuResourceMgr::GetInstance().Free(); #endif } diff --git a/core/unittest/scheduler/test_scheduler.cpp b/core/unittest/scheduler/test_scheduler.cpp index 72538113c3..c839307958 100644 --- a/core/unittest/scheduler/test_scheduler.cpp +++ b/core/unittest/scheduler/test_scheduler.cpp @@ -98,24 +98,25 @@ class SchedulerTest : public testing::Test { protected: void SetUp() override { + res_mgr_ = std::make_shared(); + ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, true, false); + ResourcePtr cpu = ResourceFactory::Create("cpu", "CPU", 0, true, false); + disk_resource_ = res_mgr_->Add(std::move(disk)); + cpu_resource_ = res_mgr_->Add(std::move(cpu)); + +#ifdef MILVUS_GPU_VERSION constexpr int64_t cache_cap = 1024 * 1024 * 1024; cache::GpuCacheMgr::GetInstance(0)->SetCapacity(cache_cap); cache::GpuCacheMgr::GetInstance(1)->SetCapacity(cache_cap); - - ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, true, false); - ResourcePtr cpu = ResourceFactory::Create("cpu", "CPU", 0, true, false); ResourcePtr gpu_0 = ResourceFactory::Create("gpu0", "GPU", 0); ResourcePtr gpu_1 = ResourceFactory::Create("gpu1", "GPU", 1); - - res_mgr_ = std::make_shared(); - disk_resource_ = res_mgr_->Add(std::move(disk)); - cpu_resource_ = res_mgr_->Add(std::move(cpu)); gpu_resource_0_ = res_mgr_->Add(std::move(gpu_0)); gpu_resource_1_ = res_mgr_->Add(std::move(gpu_1)); auto PCIE = Connection("IO", 11000.0); res_mgr_->Connect("cpu", "gpu0", PCIE); res_mgr_->Connect("cpu", "gpu1", PCIE); +#endif scheduler_ = std::make_shared(res_mgr_); @@ -138,17 +139,6 @@ class SchedulerTest : public testing::Test { std::shared_ptr scheduler_; }; -void -insert_dummy_index_into_gpu_cache(uint64_t device_id) { - MockVecIndex* mock_index = new MockVecIndex(); - mock_index->ntotal_ = 1000; - engine::VecIndexPtr index(mock_index); - - cache::DataObjPtr obj = std::static_pointer_cast(index); - - cache::GpuCacheMgr::GetInstance(device_id)->InsertItem("location", obj); -} - class SchedulerTest2 : public testing::Test { protected: void @@ -157,16 +147,13 @@ class SchedulerTest2 : public testing::Test { ResourcePtr cpu0 = ResourceFactory::Create("cpu0", "CPU", 0, true, false); ResourcePtr cpu1 = ResourceFactory::Create("cpu1", "CPU", 1, true, false); ResourcePtr cpu2 = ResourceFactory::Create("cpu2", "CPU", 2, true, false); - ResourcePtr gpu0 = ResourceFactory::Create("gpu0", "GPU", 0, true, true); - ResourcePtr gpu1 = ResourceFactory::Create("gpu1", "GPU", 1, true, true); res_mgr_ = std::make_shared(); disk_ = res_mgr_->Add(std::move(disk)); cpu_0_ = res_mgr_->Add(std::move(cpu0)); cpu_1_ = res_mgr_->Add(std::move(cpu1)); cpu_2_ = res_mgr_->Add(std::move(cpu2)); - gpu_0_ = res_mgr_->Add(std::move(gpu0)); - gpu_1_ = res_mgr_->Add(std::move(gpu1)); + auto IO = Connection("IO", 5.0); auto PCIE1 = Connection("PCIE", 11.0); auto PCIE2 = Connection("PCIE", 20.0); @@ -174,8 +161,15 @@ class SchedulerTest2 : public testing::Test { res_mgr_->Connect("cpu0", "cpu1", IO); res_mgr_->Connect("cpu1", "cpu2", IO); res_mgr_->Connect("cpu0", "cpu2", IO); + +#ifdef MILVUS_GPU_VERSION + ResourcePtr gpu0 = ResourceFactory::Create("gpu0", "GPU", 0, true, true); + ResourcePtr gpu1 = ResourceFactory::Create("gpu1", "GPU", 1, true, true); + gpu_0_ = res_mgr_->Add(std::move(gpu0)); + gpu_1_ = res_mgr_->Add(std::move(gpu1)); res_mgr_->Connect("cpu1", "gpu0", PCIE1); res_mgr_->Connect("cpu2", "gpu1", PCIE2); +#endif scheduler_ = std::make_shared(res_mgr_); diff --git a/core/unittest/server/test_cache.cpp b/core/unittest/server/test_cache.cpp index 67e9664d2c..92e09d4a26 100644 --- a/core/unittest/server/test_cache.cpp +++ b/core/unittest/server/test_cache.cpp @@ -175,6 +175,7 @@ TEST(CacheTest, CPU_CACHE_TEST) { cpu_mgr->PrintInfo(); } +#ifdef MILVUS_GPU_VERSION TEST(CacheTest, GPU_CACHE_TEST) { auto gpu_mgr = milvus::cache::GpuCacheMgr::GetInstance(0); @@ -202,6 +203,7 @@ TEST(CacheTest, GPU_CACHE_TEST) { gpu_mgr->ClearCache(); ASSERT_EQ(gpu_mgr->ItemCount(), 0); } +#endif TEST(CacheTest, INVALID_TEST) { { diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 664a08d631..791876ee8b 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -25,6 +25,8 @@ #include "utils/StringHelpFunctions.h" #include "utils/ValidationUtil.h" +#include + namespace { static constexpr uint64_t KB = 1024; @@ -63,9 +65,21 @@ TEST_F(ConfigTest, CONFIG_TEST) { int64_t port = server_config.GetInt64Value("port"); ASSERT_NE(port, 0); - server_config.SetValue("test", "2.5"); - double test = server_config.GetDoubleValue("test"); - ASSERT_EQ(test, 2.5); + server_config.SetValue("float_test", "2.5"); + double dbl = server_config.GetDoubleValue("float_test"); + ASSERT_LE(abs(dbl - 2.5), std::numeric_limits::epsilon()); + float flt = server_config.GetFloatValue("float_test"); + ASSERT_LE(abs(flt - 2.5), std::numeric_limits::epsilon()); + + server_config.SetValue("bool_test", "true"); + bool blt = server_config.GetBoolValue("bool_test"); + ASSERT_TRUE(blt); + + server_config.SetValue("int_test", "34"); + int32_t it32 = server_config.GetInt32Value("int_test"); + ASSERT_EQ(it32, 34); + int64_t it64 = server_config.GetInt64Value("int_test"); + ASSERT_EQ(it64, 34); milvus::server::ConfigNode fake; server_config.AddChild("fake", fake); @@ -236,6 +250,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(s.ok()); ASSERT_TRUE(int64_val == engine_omp_thread_num); +#ifdef MILVUS_GPU_VERSION int64_t engine_gpu_search_threshold = 800; s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)); ASSERT_TRUE(s.ok()); @@ -251,7 +266,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(s.ok()); ASSERT_TRUE(bool_val == resource_enable_gpu); -#ifdef MILVUS_GPU_VERSION int64_t gpu_cache_capacity = 1; s = config.SetGpuResourceConfigCacheCapacity(std::to_string(gpu_cache_capacity)); ASSERT_TRUE(s.ok()); @@ -389,6 +403,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetEngineConfigOmpThreadNum("10000"); ASSERT_FALSE(s.ok()); +#ifdef MILVUS_GPU_VERSION s = config.SetEngineConfigGpuSearchThreshold("-1"); ASSERT_FALSE(s.ok()); @@ -396,7 +411,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetGpuResourceConfigEnable("ok"); ASSERT_FALSE(s.ok()); -#ifdef MILVUS_GPU_VERSION s = config.SetGpuResourceConfigCacheCapacity("a"); ASSERT_FALSE(s.ok()); s = config.SetGpuResourceConfigCacheCapacity("128"); diff --git a/core/unittest/server/test_rpc.cpp b/core/unittest/server/test_rpc.cpp index 5753c68422..e04411fad3 100644 --- a/core/unittest/server/test_rpc.cpp +++ b/core/unittest/server/test_rpc.cpp @@ -313,6 +313,9 @@ TEST_F(RpcHandlerTest, TABLES_TEST) { std::vector> record_array; BuildVectors(0, VECTOR_COUNT, record_array); ::milvus::grpc::VectorIds vector_ids; + for (int64_t i = 0; i < VECTOR_COUNT; i++) { + vector_ids.add_vector_id_array(i); + } // Insert vectors // test invalid table name handler->Insert(&context, &request, &vector_ids); diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index e5884cac65..01cf713bcd 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -120,7 +120,13 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) { milvus::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result); ASSERT_EQ(result.size(), 3UL); + std::string merge_str; + milvus::server::StringHelpFunctions::MergeStringWithDelimeter(result, ",", merge_str); + ASSERT_EQ(merge_str, "a,b,c"); result.clear(); + milvus::server::StringHelpFunctions::MergeStringWithDelimeter(result, ",", merge_str); + ASSERT_TRUE(merge_str.empty()); + auto status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result); ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); @@ -211,6 +217,11 @@ TEST(UtilTest, STATUS_TEST) { str = status.ToString(); ASSERT_FALSE(str.empty()); + status = milvus::Status(milvus::DB_INVALID_PATH, "mistake"); + ASSERT_EQ(status.code(), milvus::DB_INVALID_PATH); + str = status.ToString(); + ASSERT_FALSE(str.empty()); + status = milvus::Status(milvus::DB_META_TRANSACTION_FAILED, "mistake"); ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED); str = status.ToString(); @@ -261,6 +272,10 @@ TEST(ValidationUtilTest, VALIDATE_TABLENAME_TEST) { table_name = std::string(10000, 'a'); status = milvus::server::ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME); + + table_name = ""; + status = milvus::server::ValidationUtil::ValidatePartitionName(table_name); + ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME); } TEST(ValidationUtilTest, VALIDATE_DIMENSION_TEST) { diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md index a6e5e75ea4..098f9e69a4 100644 --- a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md @@ -16,25 +16,25 @@ ### 软硬件环境 -操作系统: CentOS Linux release 7.6.1810 (Core) +操作系统：CentOS Linux release 7.6.1810 (Core) -CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz +CPU：Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz -GPU0: GeForce GTX 1080 +GPU0：GeForce GTX 1080 -GPU1: GeForce GTX 1080 +GPU1：GeForce GTX 1080 -内存: 503GB +内存：503GB -Docker版本: 18.09 +Docker版本：18.09 -NVIDIA Driver版本: 430.34 +NVIDIA Driver版本：430.34 -Milvus版本: 0.5.3 +Milvus版本：0.5.3 -SDK接口: Python 3.6.8 +SDK接口：Python 3.6.8 -pymilvus版本: 0.2.5 +pymilvus版本：0.2.5 @@ -51,7 +51,7 @@ pymilvus版本: 0.2.5 ### 测试指标 -- Query Elapsed Time: 数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量: +- Query Elapsed Time：数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量： - nq (被查询向量的数量) @@ -59,7 +59,7 @@ pymilvus版本: 0.2.5 > > 被查询向量的数量nq将按照 [1, 5, 10, 200, 400, 600, 800, 1000]的数量分组。 -- Recall: 实际返回的正确结果占总数之比 . 影响Recall的变量: +- Recall：实际返回的正确结果占总数之比。影响Recall的变量： - nq (被查询向量的数量) - topk (单条查询中最相似的K个结果) @@ -76,7 +76,7 @@ pymilvus版本: 0.2.5 ### 测试环境 -数据集: sift1b-1,000,000,000向量, 128维 +数据集：sift1b-1,000,000,000向量，128维 表格属性： @@ -143,7 +143,7 @@ search_resources: cpu, gpu0 | nq=800 | 23.24 | | nq=1000 | 27.41 | -当nq为1000时，在GPU模式下查询一条128维向量需要耗时约27毫秒。 +当nq为1000时，在CPU模式下查询一条128维向量需要耗时约27毫秒。 diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md index b50d00f9bd..daac2af545 100644 --- a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md +++ b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md @@ -139,7 +139,7 @@ topk = 100 **总结** -当nq小于1200时，查询耗时随nq的增长快速增大；当nq大于1200时，查询耗时的增大则缓慢许多。这是因为gpu_search_threshold这一参数的值被设为1200，当nq<1200时，选择CPU进行操作，否则选择GPU进行操作。与CPU。 +当nq小于1200时，查询耗时随nq的增长快速增大；当nq大于1200时，查询耗时的增大则缓慢许多。这是因为gpu_search_threshold这一参数的值被设为1200，当nq小于1200时，选择CPU进行操作，否则选择GPU进行操作。 在GPU模式下的查询耗时由两部分组成：（1）索引从CPU到GPU的拷贝时间；（2）所有分桶的查询时间。当nq小于500时，索引从CPU到GPU 的拷贝时间无法被有效均摊，此时CPU模式时一个更优的选择；当nq大于500时，选择GPU模式更合理。和CPU相比，GPU具有更多的核数和更强的算力。当nq较大时，GPU在计算上的优势能被更好地被体现。 diff --git a/ci/function/file_transfer.groovy b/tests/milvus_ann_acc/ci/function/file_transfer.groovy similarity index 100% rename from ci/function/file_transfer.groovy rename to tests/milvus_ann_acc/ci/function/file_transfer.groovy diff --git a/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy b/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy new file mode 100644 index 0000000000..c1b2b2ed64 --- /dev/null +++ b/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy @@ -0,0 +1,16 @@ +timeout(time: 7200, unit: 'MINUTES') { + try { + dir ("milvu_ann_acc") { + print "Git clone url: ${TEST_URL}:${TEST_BRANCH}" + checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]]) + print "Install requirements" + sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' + // sleep(120000) + sh "python3 main.py --suite=${params.SUITE} --host=acc-test-${env.JOB_NAME}-${env.BUILD_NUMBER}-engine.milvus.svc.cluster.local --port=19530" + } + } catch (exc) { + echo 'Milvus Ann Accuracy Test Failed !' + throw exc + } +} + diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/tests/milvus_ann_acc/ci/jenkinsfile/cleanup.groovy similarity index 100% rename from ci/jenkinsfile/cleanup_dev.groovy rename to tests/milvus_ann_acc/ci/jenkinsfile/cleanup.groovy diff --git a/tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy b/tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy new file mode 100644 index 0000000000..951bb69941 --- /dev/null +++ b/tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy @@ -0,0 +1,22 @@ +timeout(time: 30, unit: 'MINUTES') { + try { + dir ("milvus") { + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo update' + checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]]) + dir ("milvus") { + sh "helm install --wait --timeout 300 --set engine.image.tag=${IMAGE_TAG} --set expose.type=clusterIP --name acc-test-${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/sqlite_${params.IMAGE_TYPE}_values.yaml -f ci/filebeat/values.yaml --namespace milvus --version ${HELM_BRANCH} ." + } + } + // dir ("milvus") { + // checkout([$class: 'GitSCM', branches: [[name: "${env.SERVER_BRANCH}"]], userRemoteConfigs: [[url: "${env.SERVER_URL}", name: 'origin', refspec: "+refs/heads/${env.SERVER_BRANCH}:refs/remotes/origin/${env.SERVER_BRANCH}"]]]) + // dir ("milvus") { + // load "ci/jenkins/step/deploySingle2Dev.groovy" + // } + // } + } catch (exc) { + echo 'Deploy Milvus Server Failed !' + throw exc + } +} + diff --git a/ci/jenkinsfile/notify.groovy b/tests/milvus_ann_acc/ci/jenkinsfile/notify.groovy similarity index 100% rename from ci/jenkinsfile/notify.groovy rename to tests/milvus_ann_acc/ci/jenkinsfile/notify.groovy diff --git a/tests/milvus_ann_acc/ci/main_jenkinsfile b/tests/milvus_ann_acc/ci/main_jenkinsfile new file mode 100644 index 0000000000..9fdac4fc6e --- /dev/null +++ b/tests/milvus_ann_acc/ci/main_jenkinsfile @@ -0,0 +1,130 @@ +pipeline { + agent none + + options { + timestamps() + } + + parameters{ + choice choices: ['cpu', 'gpu'], description: 'cpu or gpu version', name: 'IMAGE_TYPE' + string defaultValue: '0.6.0', description: 'server image version', name: 'IMAGE_VERSION', trim: true + string defaultValue: 'suite.yaml', description: 'test suite config yaml', name: 'SUITE', trim: true + string defaultValue: '09509e53-9125-4f5d-9ce8-42855987ad67', description: 'git credentials', name: 'GIT_USER', trim: true + } + + environment { + IMAGE_TAG = "${params.IMAGE_VERSION}-${params.IMAGE_TYPE}-ubuntu18.04-release" + HELM_URL = "https://github.com/milvus-io/milvus-helm.git" + HELM_BRANCH = "0.6.0" + TEST_URL = "git@192.168.1.105:Test/milvus_ann_acc.git" + TEST_BRANCH = "0.6.0" + } + + stages { + stage("Setup env") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ + apiVersion: v1 + kind: Pod + metadata: + labels: + app: milvus + componet: test + spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + - name: hdf5-path + mountPath: /test + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config + - name: hdf5-path + flexVolume: + driver: "fstab/cifs" + fsType: "cifs" + secretRef: + name: "cifs-test-secret" + options: + networkPath: "//192.168.1.126/test" + mountOptions: "vers=1.0" + """ + } + } + + stages { + stage("Deploy Default Server") { + steps { + gitlabCommitStatus(name: 'Accuracy Test') { + container('milvus-testframework') { + script { + print "In Deploy Default Server Stage" + load "${env.WORKSPACE}/ci/jenkinsfile/deploy_default_server.groovy" + } + } + } + } + } + stage("Acc Test") { + steps { + gitlabCommitStatus(name: 'Accuracy Test') { + container('milvus-testframework') { + script { + print "In Acc test stage" + load "${env.WORKSPACE}/ci/jenkinsfile/acc_test.groovy" + } + } + } + } + } + stage ("Cleanup Env") { + steps { + gitlabCommitStatus(name: 'Cleanup Env') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy" + } + } + } + success { + script { + echo "Milvus ann-accuracy test success !" + } + } + aborted { + script { + echo "Milvus ann-accuracy test aborted !" + } + } + failure { + script { + echo "Milvus ann-accuracy test failed !" + } + } + } + } + } +} diff --git a/ci/pod_containers/milvus-testframework.yaml b/tests/milvus_ann_acc/ci/pod_containers/milvus-testframework.yaml similarity index 76% rename from ci/pod_containers/milvus-testframework.yaml rename to tests/milvus_ann_acc/ci/pod_containers/milvus-testframework.yaml index 7a98fbca8e..6b1d6c7dfd 100644 --- a/ci/pod_containers/milvus-testframework.yaml +++ b/tests/milvus_ann_acc/ci/pod_containers/milvus-testframework.yaml @@ -7,7 +7,7 @@ metadata: spec: containers: - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.1 + image: registry.zilliz.com/milvus/milvus-test:v0.2 command: - cat tty: true diff --git a/tests/milvus_ann_acc/client.py b/tests/milvus_ann_acc/client.py index de4ef17cb6..6fec829612 100644 --- a/tests/milvus_ann_acc/client.py +++ b/tests/milvus_ann_acc/client.py @@ -8,7 +8,7 @@ import numpy import sklearn.preprocessing from milvus import Milvus, IndexType, MetricType -logger = logging.getLogger("milvus_ann_acc.client") +logger = logging.getLogger("milvus_acc.client") SERVER_HOST_DEFAULT = "127.0.0.1" SERVER_PORT_DEFAULT = 19530 @@ -28,17 +28,17 @@ def time_wrapper(func): class MilvusClient(object): - def __init__(self, table_name=None, ip=None, port=None): + def __init__(self, table_name=None, host=None, port=None): self._milvus = Milvus() self._table_name = table_name try: - if not ip: + if not host: self._milvus.connect( host = SERVER_HOST_DEFAULT, port = SERVER_PORT_DEFAULT) else: self._milvus.connect( - host = ip, + host = host, port = port) except Exception as e: raise e @@ -113,7 +113,6 @@ class MilvusClient(object): X = X.astype(numpy.float32) status, results = self._milvus.search_vectors(self._table_name, top_k, nprobe, X.tolist()) self.check_status(status) - # logger.info(results[0]) ids = [] for result in results: tmp_ids = [] @@ -125,24 +124,20 @@ class MilvusClient(object): def count(self): return self._milvus.get_table_row_count(self._table_name)[1] - def delete(self, timeout=60): - logger.info("Start delete table: %s" % self._table_name) - self._milvus.delete_table(self._table_name) - i = 0 - while i < timeout: - if self.count(): - time.sleep(1) - i = i + 1 - else: - break - if i >= timeout: - logger.error("Delete table timeout") + def delete(self, table_name): + logger.info("Start delete table: %s" % table_name) + return self._milvus.delete_table(table_name) def describe(self): return self._milvus.describe_table(self._table_name) - def exists_table(self): - return self._milvus.has_table(self._table_name) + def exists_table(self, table_name): + return self._milvus.has_table(table_name) + + def get_server_version(self): + status, res = self._milvus.server_version() + self.check_status(status) + return res @time_wrapper def preload_table(self): diff --git a/tests/milvus_ann_acc/main.py b/tests/milvus_ann_acc/main.py index 308e8246c7..703303232d 100644 --- a/tests/milvus_ann_acc/main.py +++ b/tests/milvus_ann_acc/main.py @@ -1,26 +1,57 @@ - +import os +import sys import argparse +from yaml import load, dump +import logging +from logging import handlers +from client import MilvusClient +import runner + +LOG_FOLDER = "logs" +logger = logging.getLogger("milvus_acc") +formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s') +if not os.path.exists(LOG_FOLDER): + os.system('mkdir -p %s' % LOG_FOLDER) +fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'acc'), "D", 1, 10) +fileTimeHandler.suffix = "%Y%m%d.log" +fileTimeHandler.setFormatter(formatter) +logging.basicConfig(level=logging.DEBUG) +fileTimeHandler.setFormatter(formatter) +logger.addHandler(fileTimeHandler) + def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - '--dataset', - metavar='NAME', - help='the dataset to load training points from', - default='glove-100-angular', - choices=DATASETS.keys()) + "--host", + default="127.0.0.1", + help="server host") parser.add_argument( - "-k", "--count", - default=10, - type=positive_int, - help="the number of near neighbours to search for") + "--port", + default=19530, + help="server port") parser.add_argument( - '--definitions', + '--suite', metavar='FILE', - help='load algorithm definitions from FILE', - default='algos.yaml') - parser.add_argument( - '--image-tag', - default=None, - help='pull image first') \ No newline at end of file + help='load config definitions from suite_czr' + '.yaml', + default='suite_czr.yaml') + args = parser.parse_args() + if args.suite: + with open(args.suite, "r") as f: + suite = load(f) + hdf5_path = suite["hdf5_path"] + dataset_configs = suite["datasets"] + if not hdf5_path or not dataset_configs: + logger.warning("No datasets given") + sys.exit() + f.close() + for dataset_config in dataset_configs: + logger.debug(dataset_config) + milvus_instance = MilvusClient(host=args.host, port=args.port) + runner.run(milvus_instance, dataset_config, hdf5_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/milvus_ann_acc/requirements.txt b/tests/milvus_ann_acc/requirements.txt index 8c10e71b1f..1f2b337423 100644 --- a/tests/milvus_ann_acc/requirements.txt +++ b/tests/milvus_ann_acc/requirements.txt @@ -2,3 +2,8 @@ numpy==1.16.3 pymilvus>=0.2.0 scikit-learn==0.19.1 h5py==2.7.1 +influxdb==5.2.2 +pyyaml==3.12 +tableprint==0.8.0 +ansicolors==1.1.8 +scipy==1.3.1 \ No newline at end of file diff --git a/tests/milvus_ann_acc/runner.py b/tests/milvus_ann_acc/runner.py new file mode 100644 index 0000000000..88a5d24016 --- /dev/null +++ b/tests/milvus_ann_acc/runner.py @@ -0,0 +1,162 @@ +import os +import pdb +import time +import random +import sys +import logging +import h5py +import numpy +from influxdb import InfluxDBClient + +INSERT_INTERVAL = 100000 +# s +DELETE_INTERVAL_TIME = 5 +INFLUXDB_HOST = "192.168.1.194" +INFLUXDB_PORT = 8086 +INFLUXDB_USER = "admin" +INFLUXDB_PASSWD = "admin" +INFLUXDB_NAME = "test_result" +influxdb_client = InfluxDBClient(host=INFLUXDB_HOST, port=INFLUXDB_PORT, username=INFLUXDB_USER, password=INFLUXDB_PASSWD, database=INFLUXDB_NAME) + +logger = logging.getLogger("milvus_acc.runner") + + +def parse_dataset_name(dataset_name): + data_type = dataset_name.split("-")[0] + dimension = int(dataset_name.split("-")[1]) + metric = dataset_name.split("-")[-1] + # metric = dataset.attrs['distance'] + # dimension = len(dataset["train"][0]) + if metric == "euclidean": + metric_type = "l2" + elif metric == "angular": + metric_type = "ip" + return ("ann"+data_type, dimension, metric_type) + + +def get_dataset(hdf5_path, dataset_name): + file_path = os.path.join(hdf5_path, '%s.hdf5' % dataset_name) + if not os.path.exists(file_path): + raise Exception("%s not existed" % file_path) + dataset = h5py.File(file_path) + return dataset + + +def get_table_name(hdf5_path, dataset_name, index_file_size): + data_type, dimension, metric_type = parse_dataset_name(dataset_name) + dataset = get_dataset(hdf5_path, dataset_name) + table_size = len(dataset["train"]) + table_size = str(table_size // 1000000)+"m" + table_name = data_type+'_'+table_size+'_'+str(index_file_size)+'_'+str(dimension)+'_'+metric_type + return table_name + + +def recall_calc(result_ids, true_ids, top_k, recall_k): + sum_intersect_num = 0 + recall = 0.0 + for index, result_item in enumerate(result_ids): + if len(set(true_ids[index][:top_k])) != len(set(result_item)): + logger.warning("Error happened: query result length is wrong") + continue + tmp = set(true_ids[index][:recall_k]).intersection(set(result_item)) + sum_intersect_num = sum_intersect_num + len(tmp) + recall = round(sum_intersect_num / (len(result_ids) * recall_k), 4) + return recall + + +def run(milvus, config, hdf5_path, force=True): + server_version = milvus.get_server_version() + logger.info(server_version) + + for dataset_name, config_value in config.items(): + dataset = get_dataset(hdf5_path, dataset_name) + index_file_sizes = config_value["index_file_sizes"] + index_types = config_value["index_types"] + nlists = config_value["nlists"] + search_param = config_value["search_param"] + top_ks = search_param["top_ks"] + nprobes = search_param["nprobes"] + nqs = search_param["nqs"] + + for index_file_size in index_file_sizes: + table_name = get_table_name(hdf5_path, dataset_name, index_file_size) + if milvus.exists_table(table_name): + if force is True: + logger.info("Re-create table: %s" % table_name) + milvus.delete(table_name) + time.sleep(DELETE_INTERVAL_TIME) + else: + logger.warning("Table name: %s existed" % table_name) + continue + data_type, dimension, metric_type = parse_dataset_name(dataset_name) + milvus.create_table(table_name, dimension, index_file_size, metric_type) + logger.info(milvus.describe()) + insert_vectors = numpy.array(dataset["train"]) + # milvus.insert(insert_vectors) + + loops = len(insert_vectors) // INSERT_INTERVAL + 1 + for i in range(loops): + start = i*INSERT_INTERVAL + end = min((i+1)*INSERT_INTERVAL, len(insert_vectors)) + tmp_vectors = insert_vectors[start:end] + if start < end: + milvus.insert(tmp_vectors, ids=[i for i in range(start, end)]) + time.sleep(20) + row_count = milvus.count() + logger.info("Table: %s, row count: %s" % (table_name, row_count)) + if milvus.count() != len(insert_vectors): + logger.error("Table row count is not equal to insert vectors") + return + for index_type in index_types: + for nlist in nlists: + milvus.create_index(index_type, nlist) + logger.info(milvus.describe_index()) + logger.info("Start preload table: %s, index_type: %s, nlist: %s" % (table_name, index_type, nlist)) + milvus.preload_table() + true_ids = numpy.array(dataset["neighbors"]) + for nprobe in nprobes: + for nq in nqs: + query_vectors = numpy.array(dataset["test"][:nq]) + for top_k in top_ks: + rec1 = 0.0 + rec10 = 0.0 + rec100 = 0.0 + result_ids = milvus.query(query_vectors, top_k, nprobe) + logger.info("Query result: %s" % len(result_ids)) + rec1 = recall_calc(result_ids, true_ids, top_k, 1) + if top_k == 10: + rec10 = recall_calc(result_ids, true_ids, top_k, 10) + if top_k == 100: + rec10 = recall_calc(result_ids, true_ids, top_k, 10) + rec100 = recall_calc(result_ids, true_ids, top_k, 100) + avg_radio = recall_calc(result_ids, true_ids, top_k, top_k) + logger.debug("Recall_1: %s" % rec1) + logger.debug("Recall_10: %s" % rec10) + logger.debug("Recall_100: %s" % rec100) + logger.debug("Accuracy: %s" % avg_radio) + acc_record = [{ + "measurement": "accuracy", + "tags": { + "server_version": server_version, + "dataset": dataset_name, + "index_file_size": index_file_size, + "index_type": index_type, + "nlist": nlist, + "search_nprobe": nprobe, + "top_k": top_k, + "nq": len(query_vectors) + }, + # "time": time.ctime(), + "time": time.strftime("%Y-%m-%dT%H:%M:%SZ"), + "fields": { + "recall1": rec1, + "recall10": rec10, + "recall100": rec100, + "avg_radio": avg_radio + } + }] + logger.info(acc_record) + try: + res = influxdb_client.write_points(acc_record) + except Exception as e: + logger.error("Insert infuxdb failed: %s" % str(e)) diff --git a/tests/milvus_ann_acc/suite.yaml b/tests/milvus_ann_acc/suite.yaml new file mode 100644 index 0000000000..1137ccfa64 --- /dev/null +++ b/tests/milvus_ann_acc/suite.yaml @@ -0,0 +1,29 @@ +datasets: + - sift-128-euclidean: + index_file_sizes: [50, 1024] + index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [1, 32, 128, 256] + top_ks: [10] + nqs: [10000] + - glove-25-angular: + index_file_sizes: [50, 1024] + index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [1, 32, 128, 256] + top_ks: [10] + nqs: [10000] + - glove-200-angular: + index_file_sizes: [50, 1024] + index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [1, 32, 128, 256] + top_ks: [10] + nqs: [10000] +hdf5_path: /test/milvus/ann_hdf5/ \ No newline at end of file diff --git a/tests/milvus_ann_acc/suite.yaml.bak b/tests/milvus_ann_acc/suite.yaml.bak new file mode 100644 index 0000000000..7736786d03 --- /dev/null +++ b/tests/milvus_ann_acc/suite.yaml.bak @@ -0,0 +1,11 @@ +datasets: + - glove-200-angular: + index_file_sizes: [1024] + index_types: ['ivf_sq8'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [256, 400, 256] + top_ks: [100] + nqs: [10000] +hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_ann_acc/suite_czr.yaml b/tests/milvus_ann_acc/suite_czr.yaml new file mode 100644 index 0000000000..7e2b0c8708 --- /dev/null +++ b/tests/milvus_ann_acc/suite_czr.yaml @@ -0,0 +1,20 @@ +datasets: + - sift-128-euclidean: + index_file_sizes: [1024] + index_types: ['ivf_sq8', 'ivf_sq8h'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [16, 128, 1024] + top_ks: [1, 10, 100] + nqs: [10, 100, 1000] + - glove-200-angular: + index_file_sizes: [1024] + index_types: ['ivf_sq8', 'ivf_sq8h'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [16, 128, 1024] + top_ks: [1, 10, 100] + nqs: [10, 100, 1000] +hdf5_path: /test/milvus/ann_hdf5/ \ No newline at end of file diff --git a/tests/milvus_ann_acc/suite_debug.yaml b/tests/milvus_ann_acc/suite_debug.yaml new file mode 100644 index 0000000000..ca463a9c40 --- /dev/null +++ b/tests/milvus_ann_acc/suite_debug.yaml @@ -0,0 +1,10 @@ +datasets: + - sift-128-euclidean: + index_file_sizes: [1024] + index_types: ['ivf_flat'] + nlists: [16384] + search_param: + nprobes: [1, 256] + top_ks: [10] + nqs: [10000] +hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_ann_acc/test.py b/tests/milvus_ann_acc/test.py index c4fbc33195..44ffd53051 100644 --- a/tests/milvus_ann_acc/test.py +++ b/tests/milvus_ann_acc/test.py @@ -1,132 +1,33 @@ -import os -import pdb import time -import random -import sys -import h5py -import numpy -import logging -from logging import handlers +from influxdb import InfluxDBClient -from client import MilvusClient +INFLUXDB_HOST = "192.168.1.194" +INFLUXDB_PORT = 8086 +INFLUXDB_USER = "admin" +INFLUXDB_PASSWD = "admin" +INFLUXDB_NAME = "test_result" -LOG_FOLDER = "logs" -logger = logging.getLogger("milvus_ann_acc") +client = InfluxDBClient(host=INFLUXDB_HOST, port=INFLUXDB_PORT, username=INFLUXDB_USER, password=INFLUXDB_PASSWD, database=INFLUXDB_NAME) -formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s') -if not os.path.exists(LOG_FOLDER): - os.system('mkdir -p %s' % LOG_FOLDER) -fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'acc'), "D", 1, 10) -fileTimeHandler.suffix = "%Y%m%d.log" -fileTimeHandler.setFormatter(formatter) -logging.basicConfig(level=logging.DEBUG) -fileTimeHandler.setFormatter(formatter) -logger.addHandler(fileTimeHandler) - - -def get_dataset_fn(dataset_name): - file_path = "/test/milvus/ann_hdf5/" - if not os.path.exists(file_path): - raise Exception("%s not exists" % file_path) - return os.path.join(file_path, '%s.hdf5' % dataset_name) - - -def get_dataset(dataset_name): - hdf5_fn = get_dataset_fn(dataset_name) - hdf5_f = h5py.File(hdf5_fn) - return hdf5_f - - -def parse_dataset_name(dataset_name): - data_type = dataset_name.split("-")[0] - dimension = int(dataset_name.split("-")[1]) - metric = dataset_name.split("-")[-1] - # metric = dataset.attrs['distance'] - # dimension = len(dataset["train"][0]) - if metric == "euclidean": - metric_type = "l2" - elif metric == "angular": - metric_type = "ip" - return ("ann"+data_type, dimension, metric_type) - - -def get_table_name(dataset_name, index_file_size): - data_type, dimension, metric_type = parse_dataset_name(dataset_name) - dataset = get_dataset(dataset_name) - table_size = len(dataset["train"]) - table_size = str(table_size // 1000000)+"m" - table_name = data_type+'_'+table_size+'_'+str(index_file_size)+'_'+str(dimension)+'_'+metric_type - return table_name - - -def main(dataset_name, index_file_size, nlist=16384, force=False): - top_k = 10 - nprobes = [32, 128] - - dataset = get_dataset(dataset_name) - table_name = get_table_name(dataset_name, index_file_size) - m = MilvusClient(table_name) - if m.exists_table(): - if force is True: - logger.info("Re-create table: %s" % table_name) - m.delete() - time.sleep(10) - else: - logger.info("Table name: %s existed" % table_name) - return - data_type, dimension, metric_type = parse_dataset_name(dataset_name) - m.create_table(table_name, dimension, index_file_size, metric_type) - print(m.describe()) - vectors = numpy.array(dataset["train"]) - query_vectors = numpy.array(dataset["test"]) - # m.insert(vectors) - - interval = 100000 - loops = len(vectors) // interval + 1 - - for i in range(loops): - start = i*interval - end = min((i+1)*interval, len(vectors)) - tmp_vectors = vectors[start:end] - if start < end: - m.insert(tmp_vectors, ids=[i for i in range(start, end)]) - - time.sleep(60) - print(m.count()) - - for index_type in ["ivf_flat", "ivf_sq8", "ivf_sq8h"]: - m.create_index(index_type, nlist) - print(m.describe_index()) - if m.count() != len(vectors): - return - m.preload_table() - true_ids = numpy.array(dataset["neighbors"]) - for nprobe in nprobes: - print("nprobe: %s" % nprobe) - sum_radio = 0.0; avg_radio = 0.0 - result_ids = m.query(query_vectors, top_k, nprobe) - # print(result_ids[:10]) - for index, result_item in enumerate(result_ids): - if len(set(true_ids[index][:top_k])) != len(set(result_item)): - logger.info("Error happened") - # logger.info(query_vectors[index]) - # logger.info(true_ids[index][:top_k], result_item) - tmp = set(true_ids[index][:top_k]).intersection(set(result_item)) - sum_radio = sum_radio + (len(tmp) / top_k) - avg_radio = round(sum_radio / len(result_ids), 4) - logger.info(avg_radio) - m.drop_index() - - -if __name__ == "__main__": - print("glove-25-angular") - # main("sift-128-euclidean", 1024, force=True) - for index_file_size in [50, 1024]: - print("Index file size: %d" % index_file_size) - main("glove-25-angular", index_file_size, force=True) - - print("sift-128-euclidean") - for index_file_size in [50, 1024]: - print("Index file size: %d" % index_file_size) - main("sift-128-euclidean", index_file_size, force=True) - # m = MilvusClient() \ No newline at end of file +print(client.get_list_database()) +acc_record = [{ + "measurement": "accuracy", + "tags": { + "server_version": "0.4.3", + "dataset": "test", + "index_type": "test", + "nlist": 12, + "search_nprobe": 12, + "top_k": 1, + "nq": 1 + }, + "time": time.ctime(), + "fields": { + "accuracy": 0.1 + } +}] +try: + res = client.write_points(acc_record) + print(res) +except Exception as e: + print(str(e)) \ No newline at end of file diff --git a/tests/milvus_python_test/test_connect.py b/tests/milvus_python_test/test_connect.py index 143ac4d8bf..b5b55f3ee1 100644 --- a/tests/milvus_python_test/test_connect.py +++ b/tests/milvus_python_test/test_connect.py @@ -149,7 +149,8 @@ class TestConnect: milvus.connect(uri=uri_value, timeout=1) assert not milvus.connected() - def test_connect_with_multiprocess(self, args): + # disable + def _test_connect_with_multiprocess(self, args): ''' target: test uri connect with multiprocess method: set correct uri, test with multiprocessing connecting diff --git a/tests/milvus_python_test/test_mix.py b/tests/milvus_python_test/test_mix.py index 5ef9ba2cde..d9331151dd 100644 --- a/tests/milvus_python_test/test_mix.py +++ b/tests/milvus_python_test/test_mix.py @@ -25,7 +25,8 @@ index_params = {'index_type': IndexType.IVFLAT, 'nlist': 16384} class TestMixBase: - def test_search_during_createIndex(self, args): + # disable + def _test_search_during_createIndex(self, args): loops = 10000 table = gen_unique_str() query_vecs = [vectors[0], vectors[1]]