admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

solve conflicts

Browse Source 
Former-commit-id: 79d4e00b90019c196b065b2654535c689dec3900
This commit is contained in:
wxyu 2019-08-24 17:50:23 +08:00
commit e5c4476db4
23 changed files with 371 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
ci/jenkinsfile/cleanup_staging.groovy Normal file
View File

@ -0,0 +1,13 @@
try {
def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
if (!result) {
sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
}
} catch (exc) {
def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
if (!result) {
sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
}
throw exc
}

16
ci/jenkinsfile/deploy2staging.groovy Normal file
View File

@ -0,0 +1,16 @@
try {
sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus'
sh 'helm repo update'
dir ("milvus-helm") {
checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
dir ("milvus/milvus-gpu") {
sh "helm install --wait --timeout 300 --set engine.image.repository="zilliz.azurecr.cn/milvus/engine" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.4.0 ."
}
}
} catch (exc) {
echo 'Helm running failed!'
sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
throw exc
}

6
ci/jenkinsfile/dev_test.groovy
View File

@ -3,7 +3,7 @@ timeout(time: 40, unit: 'MINUTES') {
dir ("${PROJECT_NAME}_test") {
checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
sh 'python3 -m pip install -r requirements.txt'
sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local"
sh "pytest . --alluredir="test_out/dev/single/sqlite" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local"
}
// mysql database backend test
@ -16,11 +16,11 @@ timeout(time: 40, unit: 'MINUTES') {
}
dir ("milvus-helm") {
dir ("milvus/milvus-gpu") {
sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.3.1 ."
sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ."
}
}
dir ("${PROJECT_NAME}_test") {
sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local"
sh "pytest . --alluredir="test_out/dev/single/mysql" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local"
}
} catch (exc) {
echo 'Milvus Test Failed !'

31
ci/jenkinsfile/staging_test.groovy Normal file
View File

@ -0,0 +1,31 @@
timeout(time: 40, unit: 'MINUTES') {
try {
dir ("${PROJECT_NAME}_test") {
checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
sh 'python3 -m pip install -r requirements.txt'
def service_ip = sh (script: "kubectl get svc --namespace milvus-1 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim()
sh "pytest . --alluredir="test_out/staging/single/sqlite" --ip ${service_ip}"
}
// mysql database backend test
load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
if (!fileExists('milvus-helm')) {
dir ("milvus-helm") {
checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
}
}
dir ("milvus-helm") {
dir ("milvus/milvus-gpu") {
sh "helm install --wait --timeout 300 --set engine.image.repository="zilliz.azurecr.cn/milvus/engine" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ."
}
}
dir ("${PROJECT_NAME}_test") {
def service_ip = sh (script: "kubectl get svc --namespace milvus-2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim()
sh "pytest . --alluredir="test_out/staging/single/mysql" --ip ${service_ip}"
}
} catch (exc) {
echo 'Milvus Test Failed !'
throw exc
}
}

4
ci/jenkinsfile/upload_dev_test_out.groovy
View File

@ -1,8 +1,8 @@
timeout(time: 5, unit: 'MINUTES') {
dir ("${PROJECT_NAME}_test") {
if (fileExists('test_out')) {
if (fileExists('test_out/dev')) {
def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
fileTransfer.FileTransfer("test_out/dev/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
}

13
ci/jenkinsfile/upload_staging_test_out.groovy Normal file
View File

@ -0,0 +1,13 @@
timeout(time: 5, unit: 'MINUTES') {
dir ("${PROJECT_NAME}_test") {
if (fileExists('test_out/staging')) {
def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
fileTransfer.FileTransfer("test_out/staging/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
}
} else {
error("Milvus Dev Test Out directory don't exists!")
}
}
}

109
ci/nightly_main_jenkinsfile
View File

@ -208,17 +208,17 @@ spec:
}
success {
script {
echo "Milvus Single Node CI/CD success !"
echo "Milvus Deploy to Dev Single Node CI/CD success !"
}
}
aborted {
script {
echo "Milvus Single Node CI/CD aborted !"
echo "Milvus Deploy to Dev Single Node CI/CD aborted !"
}
}
failure {
script {
echo "Milvus Single Node CI/CD failure !"
echo "Milvus Deploy to Dev Single Node CI/CD failure !"
}
}
}
@ -300,17 +300,114 @@ spec:
}
success {
script {
echo "Milvus Cluster CI/CD success !"
echo "Milvus Deploy to Dev Cluster CI/CD success !"
}
}
aborted {
script {
echo "Milvus Cluster CI/CD aborted !"
echo "Milvus Deploy to Dev Cluster CI/CD aborted !"
}
}
failure {
script {
echo "Milvus Cluster CI/CD failure !"
echo "Milvus Deploy to Dev Cluster CI/CD failure !"
}
}
}
}
}
}
stage("Deploy to Staging") {
parallel {
stage("Single Node") {
agent {
kubernetes {
label 'dev-test'
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: milvus
componet: test
spec:
containers:
- name: milvus-testframework
image: registry.zilliz.com/milvus/milvus-test:v0.2
command:
- cat
tty: true
volumeMounts:
- name: kubeconf
mountPath: /root/.kube/
readOnly: true
volumes:
- name: kubeconf
secret:
secretName: aks-gpu-cluster-config
"""
}
}
stages {
stage("Deploy to Staging") {
steps {
gitlabCommitStatus(name: 'Deloy to Staging') {
container('milvus-testframework') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/deploy2staging.groovy"
}
}
}
}
}
stage("Staging Test") {
steps {
gitlabCommitStatus(name: 'Staging Test') {
container('milvus-testframework') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/staging_test.groovy"
load "${env.WORKSPACE}/ci/jenkinsfile/upload_staging_test_out.groovy"
}
}
}
}
}
stage ("Cleanup Staging") {
steps {
gitlabCommitStatus(name: 'Cleanup Staging') {
container('milvus-testframework') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
}
}
}
}
}
}
post {
always {
container('milvus-testframework') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
}
}
}
success {
script {
echo "Milvus Deploy to Staging Single Node CI/CD success !"
}
}
aborted {
script {
echo "Milvus Deploy to Staging Single Node CI/CD aborted !"
}
}
failure {
script {
echo "Milvus Deploy to Staging Single Node CI/CD failure !"
}
}
}

1
cpp/CHANGELOG.md
View File

@ -56,6 +56,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-414 - Add TaskType in Scheduler::Task
- MS-415 - Add command tasktable to dump all tasktables
- MS-418 - Update server_config.template file, set CPU compute only default
- MS-419 - Move index_file_size from IndexParam to TableSchema
## New Feature
- MS-343 - Implement ResourceMgr

4
cpp/src/db/DBImpl.cpp
View File

@ -55,7 +55,9 @@ DBImpl::DBImpl(const Options& options)
}
Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
return meta_ptr_->CreateTable(table_schema);
meta::TableSchema temp_schema = table_schema;
temp_schema.index_file_size_ *= ONE_MB;
return meta_ptr_->CreateTable(temp_schema);
}
Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) {

1
cpp/src/db/Types.h
View File

@ -24,7 +24,6 @@ typedef std::vector<QueryResult> QueryResults;
struct TableIndex {
int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;
int32_t nlist_ = 16384;
int32_t index_file_size_ = 1024; //MB
int32_t metric_type_ = (int)MetricType::L2;
};

1
cpp/src/db/Utils.cpp
View File

@ -149,7 +149,6 @@ Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) {
return index1.engine_type_ == index2.engine_type_
&& index1.nlist_ == index2.nlist_
&& index1.index_file_size_ == index2.index_file_size_
&& index1.metric_type_ == index2.metric_type_;
}

4
cpp/src/db/meta/MetaTypes.h
View File

@ -40,9 +40,9 @@ struct TableSchema {
uint16_t dimension_ = 0;
int64_t created_on_ = 0;
int64_t flag_ = 0;
int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE;
int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
int32_t nlist_ = DEFAULT_NLIST;
int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE;
int32_t metric_type_ = DEFAULT_METRIC_TYPE;
}; // TableSchema
@ -69,9 +69,9 @@ struct TableFileSchema {
std::string location_;
int64_t updated_time_ = 0;
int64_t created_on_ = 0;
int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta
int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
int32_t nlist_ = DEFAULT_NLIST; //not persist to meta
int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta
int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta
}; // TableFileSchema

36
cpp/src/db/meta/MySQLMetaImpl.cpp
View File

@ -138,9 +138,9 @@ Status MySQLMetaImpl::Initialize() {
"dimension SMALLINT NOT NULL, " <<
"created_on BIGINT NOT NULL, " <<
"flag BIGINT DEFAULT 0 NOT NULL, " <<
"index_file_size INT DEFAULT 1024 NOT NULL, " <<
"engine_type INT DEFAULT 1 NOT NULL, " <<
"nlist INT DEFAULT 16384 NOT NULL, " <<
"index_file_size INT DEFAULT 1024 NOT NULL, " <<
"metric_type INT DEFAULT 1 NOT NULL);";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
@ -407,7 +407,6 @@ Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const T
"created_on = " << created_on << ", " <<
"engine_type_ = " << index.engine_type_ << ", " <<
"nlist = " << index.nlist_ << ", " <<
"index_file_size = " << index.index_file_size_*ONE_MB << ", " <<
"metric_type = " << index.metric_type_ << " " <<
"WHERE id = " << quote << table_id << ";";
@ -504,7 +503,6 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex
index.engine_type_ = resRow["engine_type"];
index.nlist_ = resRow["nlist"];
index.index_file_size_ = resRow["index_file_size"]/ONE_MB;
index.metric_type_ = resRow["metric_type"];
} else {
return Status::NotFound("Table " + table_id + " not found");
@ -694,12 +692,12 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) {
table_schema.dimension_ = resRow["dimension"];
table_schema.index_file_size_ = resRow["index_file_size"];
table_schema.engine_type_ = resRow["engine_type"];
table_schema.nlist_ = resRow["nlist"];
table_schema.index_file_size_ = resRow["index_file_size"];
table_schema.metric_type_ = resRow["metric_type"];
} else {
return Status::NotFound("Table " + table_schema.table_id_ + " not found");
@ -790,12 +788,12 @@ Status MySQLMetaImpl::AllTables(std::vector<TableSchema> &table_schema_array) {
table_schema.dimension_ = resRow["dimension"];
table_schema.index_file_size_ = resRow["index_file_size"];
table_schema.engine_type_ = resRow["engine_type"];
table_schema.nlist_ = resRow["nlist"];
table_schema.index_file_size_ = resRow["index_file_size"];
table_schema.metric_type_ = resRow["metric_type"];
table_schema_array.emplace_back(table_schema);
@ -833,9 +831,9 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
file_schema.row_count_ = 0;
file_schema.created_on_ = utils::GetMicroSecTimeStamp();
file_schema.updated_time_ = file_schema.created_on_;
file_schema.index_file_size_ = table_schema.index_file_size_;
file_schema.engine_type_ = table_schema.engine_type_;
file_schema.nlist_ = table_schema.nlist_;
file_schema.index_file_size_ = table_schema.index_file_size_;
file_schema.metric_type_ = table_schema.metric_type_;
utils::GetTableFilePath(options_, file_schema);
@ -950,10 +948,10 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) {
groups[table_file.table_id_] = table_schema;
}
table_file.nlist_ = groups[table_file.table_id_].nlist_;
table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_;
table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
table_file.dimension_ = groups[table_file.table_id_].dimension_;
table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_;
table_file.nlist_ = groups[table_file.table_id_].nlist_;
table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
utils::GetTableFilePath(options_, table_file);
@ -1043,12 +1041,12 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
resRow["table_id"].to_string(table_id_str);
table_file.table_id_ = table_id_str;
table_file.index_file_size_ = table_schema.index_file_size_;
table_file.engine_type_ = resRow["engine_type"];
table_file.nlist_ = table_schema.nlist_;
table_file.index_file_size_ = table_schema.index_file_size_;
table_file.metric_type_ = table_schema.metric_type_;
std::string file_id;
@ -1157,12 +1155,12 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
resRow["table_id"].to_string(table_id_str);
table_file.table_id_ = table_id_str;
table_file.index_file_size_ = table_schema.index_file_size_;
table_file.engine_type_ = resRow["engine_type"];
table_file.nlist_ = table_schema.nlist_;
table_file.index_file_size_ = table_schema.index_file_size_;
table_file.metric_type_ = table_schema.metric_type_;
std::string file_id;
@ -1259,12 +1257,12 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
table_file.date_ = resRow["date"];
table_file.index_file_size_ = table_schema.index_file_size_;
table_file.engine_type_ = resRow["engine_type"];
table_file.nlist_ = table_schema.nlist_;
table_file.index_file_size_ = table_schema.index_file_size_;
table_file.metric_type_ = table_schema.metric_type_;
table_file.created_on_ = resRow["created_on"];
@ -1344,12 +1342,12 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
file_schema.table_id_ = table_id;
file_schema.index_file_size_ = table_schema.index_file_size_;
file_schema.engine_type_ = resRow["engine_type"];
file_schema.nlist_ = table_schema.nlist_;
file_schema.index_file_size_ = table_schema.index_file_size_;
file_schema.metric_type_ = table_schema.metric_type_;
std::string file_id;

56
cpp/src/db/meta/SqliteMetaImpl.cpp
View File

@ -45,9 +45,9 @@ inline auto StoragePrototype(const std::string &path) {
make_column("dimension", &TableSchema::dimension_),
make_column("created_on", &TableSchema::created_on_),
make_column("flag", &TableSchema::flag_, default_value(0)),
make_column("index_file_size", &TableSchema::index_file_size_),
make_column("engine_type", &TableSchema::engine_type_),
make_column("nlist", &TableSchema::nlist_),
make_column("index_file_size", &TableSchema::index_file_size_),
make_column("metric_type", &TableSchema::metric_type_)),
make_table("TableFiles",
make_column("id", &TableFileSchema::id_, primary_key()),
@ -251,9 +251,9 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) {
&TableSchema::dimension_,
&TableSchema::created_on_,
&TableSchema::flag_,
&TableSchema::index_file_size_,
&TableSchema::engine_type_,
&TableSchema::nlist_,
&TableSchema::index_file_size_,
&TableSchema::metric_type_),
where(c(&TableSchema::table_id_) == table_schema.table_id_
and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE));
@ -264,9 +264,9 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) {
table_schema.dimension_ = std::get<2>(groups[0]);
table_schema.created_on_ = std::get<3>(groups[0]);
table_schema.flag_ = std::get<4>(groups[0]);
table_schema.engine_type_ = std::get<5>(groups[0]);
table_schema.nlist_ = std::get<6>(groups[0]);
table_schema.index_file_size_ = std::get<7>(groups[0]);
table_schema.index_file_size_ = std::get<5>(groups[0]);
table_schema.engine_type_ = std::get<6>(groups[0]);
table_schema.nlist_ = std::get<7>(groups[0]);
table_schema.metric_type_ = std::get<8>(groups[0]);
} else {
return Status::NotFound("Table " + table_schema.table_id_ + " not found");
@ -344,7 +344,8 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const
&TableSchema::state_,
&TableSchema::dimension_,
&TableSchema::created_on_,
&TableSchema::flag_),
&TableSchema::flag_,
&TableSchema::index_file_size_),
where(c(&TableSchema::table_id_) == table_id
and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE));
@ -356,9 +357,9 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const
table_schema.dimension_ = std::get<2>(tables[0]);
table_schema.created_on_ = std::get<3>(tables[0]);
table_schema.flag_ = std::get<4>(tables[0]);
table_schema.index_file_size_ = std::get<5>(tables[0]);
table_schema.engine_type_ = index.engine_type_;
table_schema.nlist_ = index.nlist_;
table_schema.index_file_size_ = index.index_file_size_*ONE_MB;
table_schema.metric_type_ = index.metric_type_;
ConnectorPtr->update(table_schema);
@ -420,8 +421,7 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde
if (groups.size() == 1) {
index.engine_type_ = std::get<0>(groups[0]);
index.nlist_ = std::get<1>(groups[0]);
index.index_file_size_ = std::get<2>(groups[0])/ONE_MB;
index.metric_type_ = std::get<3>(groups[0]);
index.metric_type_ = std::get<2>(groups[0]);
} else {
return Status::NotFound("Table " + table_id + " not found");
}
@ -499,9 +499,9 @@ Status SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
&TableSchema::dimension_,
&TableSchema::created_on_,
&TableSchema::flag_,
&TableSchema::index_file_size_,
&TableSchema::engine_type_,
&TableSchema::nlist_,
&TableSchema::index_file_size_,
&TableSchema::metric_type_),
where(c(&TableSchema::state_) != (int)TableSchema::TO_DELETE));
for (auto &table : selected) {
@ -511,9 +511,9 @@ Status SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
schema.dimension_ = std::get<2>(table);
schema.created_on_ = std::get<3>(table);
schema.flag_ = std::get<4>(table);
schema.engine_type_ = std::get<5>(table);
schema.nlist_ = std::get<6>(table);
schema.index_file_size_ = std::get<7>(table);
schema.index_file_size_ = std::get<5>(table);
schema.engine_type_ = std::get<6>(table);
schema.nlist_ = std::get<7>(table);
schema.metric_type_ = std::get<8>(table);
table_schema_array.emplace_back(schema);
@ -546,9 +546,9 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
file_schema.row_count_ = 0;
file_schema.created_on_ = utils::GetMicroSecTimeStamp();
file_schema.updated_time_ = file_schema.created_on_;
file_schema.index_file_size_ = table_schema.index_file_size_;
file_schema.engine_type_ = table_schema.engine_type_;
file_schema.nlist_ = table_schema.nlist_;
file_schema.index_file_size_ = table_schema.index_file_size_;
file_schema.metric_type_ = table_schema.metric_type_;
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
@ -609,10 +609,10 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
}
groups[table_file.table_id_] = table_schema;
}
table_file.nlist_ = groups[table_file.table_id_].nlist_;
table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_;
table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
table_file.dimension_ = groups[table_file.table_id_].dimension_;
table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_;
table_file.nlist_ = groups[table_file.table_id_].nlist_;
table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
files.push_back(table_file);
}
@ -662,9 +662,11 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
table_file.row_count_ = std::get<5>(file);
table_file.date_ = std::get<6>(file);
table_file.engine_type_ = std::get<7>(file);
table_file.metric_type_ = table_schema.metric_type_;
table_file.nlist_ = table_schema.nlist_;
table_file.dimension_ = table_schema.dimension_;
table_file.index_file_size_ = table_schema.index_file_size_;
table_file.nlist_ = table_schema.nlist_;
table_file.metric_type_ = table_schema.metric_type_;
utils::GetTableFilePath(options_, table_file);
auto dateItr = files.find(table_file.date_);
if (dateItr == files.end()) {
@ -705,10 +707,10 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
table_file.row_count_ = std::get<5>(file);
table_file.date_ = std::get<6>(file);
table_file.engine_type_ = std::get<7>(file);
table_file.nlist_ = table_schema.nlist_;
table_file.index_file_size_ = table_schema.index_file_size_;
table_file.metric_type_ = table_schema.metric_type_;
table_file.dimension_ = table_schema.dimension_;
table_file.index_file_size_ = table_schema.index_file_size_;
table_file.nlist_ = table_schema.nlist_;
table_file.metric_type_ = table_schema.metric_type_;
utils::GetTableFilePath(options_, table_file);
auto dateItr = files.find(table_file.date_);
@ -786,8 +788,8 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
table_file.date_ = std::get<6>(file);
table_file.engine_type_ = std::get<7>(file);
table_file.dimension_ = table_schema.dimension_;
table_file.nlist_ = table_schema.nlist_;
table_file.index_file_size_ = table_schema.index_file_size_;
table_file.nlist_ = table_schema.nlist_;
table_file.metric_type_ = table_schema.metric_type_;
utils::GetTableFilePath(options_, table_file);
@ -848,8 +850,8 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
table_file.date_ = std::get<6>(file);
table_file.created_on_ = std::get<7>(file);
table_file.dimension_ = table_schema.dimension_;
table_file.nlist_ = table_schema.nlist_;
table_file.index_file_size_ = table_schema.index_file_size_;
table_file.nlist_ = table_schema.nlist_;
table_file.metric_type_ = table_schema.metric_type_;
utils::GetTableFilePath(options_, table_file);
@ -900,11 +902,11 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
file_schema.row_count_ = std::get<4>(file);
file_schema.date_ = std::get<5>(file);
file_schema.engine_type_ = std::get<6>(file);
file_schema.nlist_ = table_schema.nlist_;
file_schema.index_file_size_ = table_schema.index_file_size_;
file_schema.metric_type_ = table_schema.metric_type_;
file_schema.created_on_ = std::get<7>(file);
file_schema.dimension_ = table_schema.dimension_;
file_schema.index_file_size_ = table_schema.index_file_size_;
file_schema.nlist_ = table_schema.nlist_;
file_schema.metric_type_ = table_schema.metric_type_;
utils::GetTableFilePath(options_, file_schema);

192
cpp/src/grpc/gen-milvus/milvus.pb.cc
View File

@ -366,6 +366,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT
~0u, // no _weak_field_map_
PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, table_name_),
PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, dimension_),
PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, index_file_size_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Range, _internal_metadata_),
~0u, // no _extensions_
@ -459,7 +460,6 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT
~0u, // no _weak_field_map_
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_type_),
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, nlist_),
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_file_size_),
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, metric_type_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, _internal_metadata_),
@ -479,19 +479,19 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT
static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
{ 0, -1, sizeof(::milvus::grpc::TableName)},
{ 7, -1, sizeof(::milvus::grpc::TableSchema)},
{ 14, -1, sizeof(::milvus::grpc::Range)},
{ 21, -1, sizeof(::milvus::grpc::RowRecord)},
{ 27, -1, sizeof(::milvus::grpc::InsertParam)},
{ 35, -1, sizeof(::milvus::grpc::VectorIds)},
{ 42, -1, sizeof(::milvus::grpc::SearchParam)},
{ 52, -1, sizeof(::milvus::grpc::SearchInFilesParam)},
{ 59, -1, sizeof(::milvus::grpc::QueryResult)},
{ 66, -1, sizeof(::milvus::grpc::TopKQueryResult)},
{ 73, -1, sizeof(::milvus::grpc::StringReply)},
{ 80, -1, sizeof(::milvus::grpc::BoolReply)},
{ 87, -1, sizeof(::milvus::grpc::TableRowCount)},
{ 94, -1, sizeof(::milvus::grpc::Command)},
{ 100, -1, sizeof(::milvus::grpc::Index)},
{ 15, -1, sizeof(::milvus::grpc::Range)},
{ 22, -1, sizeof(::milvus::grpc::RowRecord)},
{ 28, -1, sizeof(::milvus::grpc::InsertParam)},
{ 36, -1, sizeof(::milvus::grpc::VectorIds)},
{ 43, -1, sizeof(::milvus::grpc::SearchParam)},
{ 53, -1, sizeof(::milvus::grpc::SearchInFilesParam)},
{ 60, -1, sizeof(::milvus::grpc::QueryResult)},
{ 67, -1, sizeof(::milvus::grpc::TopKQueryResult)},
{ 74, -1, sizeof(::milvus::grpc::StringReply)},
{ 81, -1, sizeof(::milvus::grpc::BoolReply)},
{ 88, -1, sizeof(::milvus::grpc::TableRowCount)},
{ 95, -1, sizeof(::milvus::grpc::Command)},
{ 101, -1, sizeof(::milvus::grpc::Index)},
{ 109, -1, sizeof(::milvus::grpc::IndexParam)},
{ 116, -1, sizeof(::milvus::grpc::DeleteByRangeParam)},
};
@ -519,35 +519,35 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] =
const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) =
"\n\014milvus.proto\022\013milvus.grpc\032\014status.prot"
"o\"D\n\tTableName\022#\n\006status\030\001 \001(\0132\023.milvus."
"grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"L\n\013Table"
"grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"e\n\013Table"
"Schema\022*\n\ntable_name\030\001 \001(\0132\026.milvus.grpc"
".TableName\022\021\n\tdimension\030\002 \001(\003\"/\n\005Range\022\023"
"\n\013start_value\030\001 \001(\t\022\021\n\tend_value\030\002 \001(\t\" "
"\n\tRowRecord\022\023\n\013vector_data\030\001 \003(\002\"i\n\013Inse"
"rtParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row_recor"
"d_array\030\002 \003(\0132\026.milvus.grpc.RowRecord\022\024\n"
"\014row_id_array\030\003 \003(\003\"I\n\tVectorIds\022#\n\006stat"
"us\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017vector_"
"id_array\030\002 \003(\003\"\242\001\n\013SearchParam\022\022\n\ntable_"
"name\030\001 \001(\t\0222\n\022query_record_array\030\002 \003(\0132\026"
".milvus.grpc.RowRecord\022-\n\021query_range_ar"
"ray\030\003 \003(\0132\022.milvus.grpc.Range\022\014\n\004topk\030\004 "
"\001(\003\022\016\n\006nprobe\030\005 \001(\003\"[\n\022SearchInFilesPara"
"m\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014search_param"
"\030\002 \001(\0132\030.milvus.grpc.SearchParam\"+\n\013Quer"
"yResult\022\n\n\002id\030\001 \001(\003\022\020\n\010distance\030\002 \001(\001\"m\n"
"\017TopKQueryResult\022#\n\006status\030\001 \001(\0132\023.milvu"
"s.grpc.Status\0225\n\023query_result_arrays\030\002 \003"
"(\0132\030.milvus.grpc.QueryResult\"H\n\013StringRe"
"ply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status"
"\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolReply\022#\n\006s"
"tatus\030\001 \001(\0132\023.milvus.grpc.Status\022\022\n\nbool"
"_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006status\030"
"\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017table_row_"
"count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(\t\"X\n\005I"
"ndex\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\022"
"\027\n\017index_file_size\030\003 \001(\005\022\023\n\013metric_type\030"
"\004 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132"
".TableName\022\021\n\tdimension\030\002 \001(\003\022\027\n\017index_f"
"ile_size\030\003 \001(\003\"/\n\005Range\022\023\n\013start_value\030\001"
" \001(\t\022\021\n\tend_value\030\002 \001(\t\" \n\tRowRecord\022\023\n\013"
"vector_data\030\001 \003(\002\"i\n\013InsertParam\022\022\n\ntabl"
"e_name\030\001 \001(\t\0220\n\020row_record_array\030\002 \003(\0132\026"
".milvus.grpc.RowRecord\022\024\n\014row_id_array\030\003"
" \003(\003\"I\n\tVectorIds\022#\n\006status\030\001 \001(\0132\023.milv"
"us.grpc.Status\022\027\n\017vector_id_array\030\002 \003(\003\""
"\242\001\n\013SearchParam\022\022\n\ntable_name\030\001 \001(\t\0222\n\022q"
"uery_record_array\030\002 \003(\0132\026.milvus.grpc.Ro"
"wRecord\022-\n\021query_range_array\030\003 \003(\0132\022.mil"
"vus.grpc.Range\022\014\n\004topk\030\004 \001(\003\022\016\n\006nprobe\030\005"
" \001(\003\"[\n\022SearchInFilesParam\022\025\n\rfile_id_ar"
"ray\030\001 \003(\t\022.\n\014search_param\030\002 \001(\0132\030.milvus"
".grpc.SearchParam\"+\n\013QueryResult\022\n\n\002id\030\001"
" \001(\003\022\020\n\010distance\030\002 \001(\001\"m\n\017TopKQueryResul"
"t\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\0225"
"\n\023query_result_arrays\030\002 \003(\0132\030.milvus.grp"
"c.QueryResult\"H\n\013StringReply\022#\n\006status\030\001"
" \001(\0132\023.milvus.grpc.Status\022\024\n\014string_repl"
"y\030\002 \001(\t\"D\n\tBoolReply\022#\n\006status\030\001 \001(\0132\023.m"
"ilvus.grpc.Status\022\022\n\nbool_reply\030\002 \001(\010\"M\n"
"\rTableRowCount\022#\n\006status\030\001 \001(\0132\023.milvus."
"grpc.Status\022\027\n\017table_row_count\030\002 \001(\003\"\026\n\007"
"Command\022\013\n\003cmd\030\001 \001(\t\"\?\n\005Index\022\022\n\nindex_t"
"ype\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\022\023\n\013metric_type\030"
"\003 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132"
"\026.milvus.grpc.TableName\022!\n\005index\030\002 \001(\0132\022"
".milvus.grpc.Index\"K\n\022DeleteByRangeParam"
"\022!\n\005range\030\001 \001(\0132\022.milvus.grpc.Range\022\022\n\nt"
@ -978,15 +978,17 @@ TableSchema::TableSchema(const TableSchema& from)
} else {
table_name_ = nullptr;
}
dimension_ = from.dimension_;
::memcpy(&dimension_, &from.dimension_,
static_cast<size_t>(reinterpret_cast<char*>(&index_file_size_) -
reinterpret_cast<char*>(&dimension_)) + sizeof(index_file_size_));
// @@protoc_insertion_point(copy_constructor:milvus.grpc.TableSchema)
}
void TableSchema::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_TableSchema_milvus_2eproto.base);
::memset(&table_name_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&dimension_) -
reinterpret_cast<char*>(&table_name_)) + sizeof(dimension_));
reinterpret_cast<char*>(&index_file_size_) -
reinterpret_cast<char*>(&table_name_)) + sizeof(index_file_size_));
}
TableSchema::~TableSchema() {
@ -1017,7 +1019,9 @@ void TableSchema::Clear() {
delete table_name_;
}
table_name_ = nullptr;
dimension_ = PROTOBUF_LONGLONG(0);
::memset(&dimension_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&index_file_size_) -
reinterpret_cast<char*>(&dimension_)) + sizeof(index_file_size_));
_internal_metadata_.Clear();
}
@ -1043,6 +1047,13 @@ const char* TableSchema::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID
CHK_(ptr);
} else goto handle_unusual;
continue;
// int64 index_file_size = 3;
case 3:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) {
index_file_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
CHK_(ptr);
} else goto handle_unusual;
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
@ -1097,6 +1108,19 @@ bool TableSchema::MergePartialFromCodedStream(
break;
}
// int64 index_file_size = 3;
case 3: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>(
input, &index_file_size_)));
} else {
goto handle_unusual;
}
break;
}
default: {
handle_unusual:
if (tag == 0) {
@ -1135,6 +1159,11 @@ void TableSchema::SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(2, this->dimension(), output);
}
// int64 index_file_size = 3;
if (this->index_file_size() != 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->index_file_size(), output);
}
if (_internal_metadata_.have_unknown_fields()) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
_internal_metadata_.unknown_fields(), output);
@ -1160,6 +1189,11 @@ void TableSchema::SerializeWithCachedSizes(
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(2, this->dimension(), target);
}
// int64 index_file_size = 3;
if (this->index_file_size() != 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->index_file_size(), target);
}
if (_internal_metadata_.have_unknown_fields()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields(), target);
@ -1195,6 +1229,13 @@ size_t TableSchema::ByteSizeLong() const {
this->dimension());
}
// int64 index_file_size = 3;
if (this->index_file_size() != 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size(
this->index_file_size());
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
@ -1228,6 +1269,9 @@ void TableSchema::MergeFrom(const TableSchema& from) {
if (from.dimension() != 0) {
set_dimension(from.dimension());
}
if (from.index_file_size() != 0) {
set_index_file_size(from.index_file_size());
}
}
void TableSchema::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
@ -1253,6 +1297,7 @@ void TableSchema::InternalSwap(TableSchema* other) {
_internal_metadata_.Swap(&other->_internal_metadata_);
swap(table_name_, other->table_name_);
swap(dimension_, other->dimension_);
swap(index_file_size_, other->index_file_size_);
}
::PROTOBUF_NAMESPACE_ID::Metadata TableSchema::GetMetadata() const {
@ -5331,16 +5376,9 @@ const char* Index::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::inte
CHK_(ptr);
} else goto handle_unusual;
continue;
// int32 index_file_size = 3;
// int32 metric_type = 3;
case 3:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) {
index_file_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
CHK_(ptr);
} else goto handle_unusual;
continue;
// int32 metric_type = 4;
case 4:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) {
metric_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
CHK_(ptr);
} else goto handle_unusual;
@ -5401,23 +5439,10 @@ bool Index::MergePartialFromCodedStream(
break;
}
// int32 index_file_size = 3;
// int32 metric_type = 3;
case 3: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>(
input, &index_file_size_)));
} else {
goto handle_unusual;
}
break;
}
// int32 metric_type = 4;
case 4: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>(
input, &metric_type_)));
@ -5464,14 +5489,9 @@ void Index::SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(2, this->nlist(), output);
}
// int32 index_file_size = 3;
if (this->index_file_size() != 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->index_file_size(), output);
}
// int32 metric_type = 4;
// int32 metric_type = 3;
if (this->metric_type() != 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(4, this->metric_type(), output);
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->metric_type(), output);
}
if (_internal_metadata_.have_unknown_fields()) {
@ -5497,14 +5517,9 @@ void Index::SerializeWithCachedSizes(
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->nlist(), target);
}
// int32 index_file_size = 3;
if (this->index_file_size() != 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->index_file_size(), target);
}
// int32 metric_type = 4;
// int32 metric_type = 3;
if (this->metric_type() != 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(4, this->metric_type(), target);
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->metric_type(), target);
}
if (_internal_metadata_.have_unknown_fields()) {
@ -5542,14 +5557,7 @@ size_t Index::ByteSizeLong() const {
this->nlist());
}
// int32 index_file_size = 3;
if (this->index_file_size() != 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->index_file_size());
}
// int32 metric_type = 4;
// int32 metric_type = 3;
if (this->metric_type() != 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
@ -5589,9 +5597,6 @@ void Index::MergeFrom(const Index& from) {
if (from.nlist() != 0) {
set_nlist(from.nlist());
}
if (from.index_file_size() != 0) {
set_index_file_size(from.index_file_size());
}
if (from.metric_type() != 0) {
set_metric_type(from.metric_type());
}
@ -5620,7 +5625,6 @@ void Index::InternalSwap(Index* other) {
_internal_metadata_.Swap(&other->_internal_metadata_);
swap(index_type_, other->index_type_);
swap(nlist_, other->nlist_);
swap(index_file_size_, other->index_file_size_);
swap(metric_type_, other->metric_type_);
}

48
cpp/src/grpc/gen-milvus/milvus.pb.h
View File

@ -396,6 +396,7 @@ class TableSchema :
enum : int {
kTableNameFieldNumber = 1,
kDimensionFieldNumber = 2,
kIndexFileSizeFieldNumber = 3,
};
// .milvus.grpc.TableName table_name = 1;
bool has_table_name() const;
@ -410,6 +411,11 @@ class TableSchema :
::PROTOBUF_NAMESPACE_ID::int64 dimension() const;
void set_dimension(::PROTOBUF_NAMESPACE_ID::int64 value);
// int64 index_file_size = 3;
void clear_index_file_size();
::PROTOBUF_NAMESPACE_ID::int64 index_file_size() const;
void set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 value);
// @@protoc_insertion_point(class_scope:milvus.grpc.TableSchema)
private:
class _Internal;
@ -417,6 +423,7 @@ class TableSchema :
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::milvus::grpc::TableName* table_name_;
::PROTOBUF_NAMESPACE_ID::int64 dimension_;
::PROTOBUF_NAMESPACE_ID::int64 index_file_size_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_milvus_2eproto;
};
@ -2318,8 +2325,7 @@ class Index :
enum : int {
kIndexTypeFieldNumber = 1,
kNlistFieldNumber = 2,
kIndexFileSizeFieldNumber = 3,
kMetricTypeFieldNumber = 4,
kMetricTypeFieldNumber = 3,
};
// int32 index_type = 1;
void clear_index_type();
@ -2331,12 +2337,7 @@ class Index :
::PROTOBUF_NAMESPACE_ID::int32 nlist() const;
void set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value);
// int32 index_file_size = 3;
void clear_index_file_size();
::PROTOBUF_NAMESPACE_ID::int32 index_file_size() const;
void set_index_file_size(::PROTOBUF_NAMESPACE_ID::int32 value);
// int32 metric_type = 4;
// int32 metric_type = 3;
void clear_metric_type();
::PROTOBUF_NAMESPACE_ID::int32 metric_type() const;
void set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value);
@ -2348,7 +2349,6 @@ class Index :
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::int32 index_type_;
::PROTOBUF_NAMESPACE_ID::int32 nlist_;
::PROTOBUF_NAMESPACE_ID::int32 index_file_size_;
::PROTOBUF_NAMESPACE_ID::int32 metric_type_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_milvus_2eproto;
@ -2820,6 +2820,20 @@ inline void TableSchema::set_dimension(::PROTOBUF_NAMESPACE_ID::int64 value) {
// @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.dimension)
}
// int64 index_file_size = 3;
inline void TableSchema::clear_index_file_size() {
index_file_size_ = PROTOBUF_LONGLONG(0);
}
inline ::PROTOBUF_NAMESPACE_ID::int64 TableSchema::index_file_size() const {
// @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.index_file_size)
return index_file_size_;
}
inline void TableSchema::set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 value) {
index_file_size_ = value;
// @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.index_file_size)
}
// -------------------------------------------------------------------
// Range
@ -3841,21 +3855,7 @@ inline void Index::set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value) {
// @@protoc_insertion_point(field_set:milvus.grpc.Index.nlist)
}
// int32 index_file_size = 3;
inline void Index::clear_index_file_size() {
index_file_size_ = 0;
}
inline ::PROTOBUF_NAMESPACE_ID::int32 Index::index_file_size() const {
// @@protoc_insertion_point(field_get:milvus.grpc.Index.index_file_size)
return index_file_size_;
}
inline void Index::set_index_file_size(::PROTOBUF_NAMESPACE_ID::int32 value) {
index_file_size_ = value;
// @@protoc_insertion_point(field_set:milvus.grpc.Index.index_file_size)
}
// int32 metric_type = 4;
// int32 metric_type = 3;
inline void Index::clear_metric_type() {
metric_type_ = 0;
}

4
cpp/src/grpc/milvus.proto
View File

@ -18,6 +18,7 @@ message TableName {
message TableSchema {
TableName table_name = 1;
int64 dimension = 2;
int64 index_file_size = 3;
}
/**
@ -126,8 +127,7 @@ message Command {
message Index {
int32 index_type = 1;
int32 nlist = 2;
int32 index_file_size = 3;
int32 metric_type = 4;
int32 metric_type = 3;
}
/**

3
cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp
View File

@ -22,6 +22,7 @@ namespace {
const std::string TABLE_NAME = GetTableName();
constexpr int64_t TABLE_DIMENSION = 512;
constexpr int64_t TABLE_INDEX_FILE_SIZE = 768;
constexpr int64_t BATCH_ROW_COUNT = 100000;
constexpr int64_t NQ = 10;
constexpr int64_t TOP_K = 10;
@ -94,6 +95,7 @@ namespace {
TableSchema tb_schema;
tb_schema.table_name = TABLE_NAME;
tb_schema.dimension = TABLE_DIMENSION;
tb_schema.index_file_size = TABLE_INDEX_FILE_SIZE;
return tb_schema;
}
@ -286,7 +288,6 @@ ClientTest::Test(const std::string& address, const std::string& port) {
index.table_name = TABLE_NAME;
index.index_type = IndexType::gpu_ivfflat;
index.nlist = 1000;
index.index_file_size = 1024;
index.metric_type = 1;
Status stat = conn->CreateIndex(index);
std::cout << "CreateIndex function call status: " << stat.ToString() << std::endl;

4
cpp/src/sdk/grpc/ClientProxy.cpp
View File

@ -83,6 +83,7 @@ ClientProxy::CreateTable(const TableSchema &param) {
::milvus::grpc::TableSchema schema;
schema.mutable_table_name()->set_table_name(param.table_name);
schema.set_dimension(param.dimension);
schema.set_index_file_size(param.index_file_size);
return client_ptr_->CreateTable(schema);
} catch (std::exception &ex) {
@ -119,7 +120,6 @@ ClientProxy::CreateIndex(const IndexParam &index_param) {
index_param.table_name);
grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type);
grpc_index_param.mutable_index()->set_nlist(index_param.nlist);
grpc_index_param.mutable_index()->set_index_file_size(index_param.index_file_size);
grpc_index_param.mutable_index()->set_metric_type(index_param.metric_type);
return client_ptr_->CreateIndex(grpc_index_param);
@ -272,6 +272,7 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
table_schema.table_name = grpc_schema.table_name().table_name();
table_schema.dimension = grpc_schema.dimension();
table_schema.index_file_size = grpc_schema.index_file_size();
return status;
} catch (std::exception &ex) {
@ -362,7 +363,6 @@ ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_para
Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param);
index_param.index_type = (IndexType)(grpc_index_param.mutable_index()->index_type());
index_param.nlist = grpc_index_param.mutable_index()->nlist();
index_param.index_file_size = grpc_index_param.mutable_index()->index_file_size();
index_param.metric_type = grpc_index_param.mutable_index()->metric_type();
return status;

4
cpp/src/sdk/include/MilvusApi.h
View File

@ -35,9 +35,8 @@ struct ConnectParam {
*/
struct TableSchema {
std::string table_name; ///< Table name
IndexType index_type = IndexType::invalid; ///< Index type
int64_t dimension = 0; ///< Vector dimension, must be a positive value
bool store_raw_vector = false; ///< Is vector raw data stored in the table
int64_t index_file_size = 0; ///< Index file size, must be a positive value
};
/**
@ -78,7 +77,6 @@ struct IndexParam {
std::string table_name;
IndexType index_type;
int32_t nlist;
int32_t index_file_size;
int32_t metric_type;
};

15
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
View File

@ -134,10 +134,16 @@ CreateTableTask::OnExecute() {
return SetError(res, "Invalid table dimension: " + std::to_string(schema_.dimension()));
}
res = ValidationUtil::ValidateTableIndexFileSize(schema_.index_file_size());
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid index file size: " + std::to_string(schema_.index_file_size()));
}
//step 2: construct table schema
engine::meta::TableSchema table_info;
table_info.dimension_ = (uint16_t) schema_.dimension();
table_info.table_id_ = schema_.table_name().table_name();
table_info.dimension_ = (uint16_t) schema_.dimension();
table_info.index_file_size_ = schema_.index_file_size();
//step 3: create table
engine::Status stat = DBWrapper::DB()->CreateTable(table_info);
@ -246,16 +252,10 @@ CreateIndexTask::OnExecute() {
return SetError(res, "Invalid index metric type: " + std::to_string(index_param_.mutable_index()->metric_type()));
}
res = ValidationUtil::ValidateTableIndexFileSize(index_param_.mutable_index()->index_file_size());
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid index file size: " + std::to_string(index_param_.mutable_index()->index_file_size()));
}
//step 2: check table existence
engine::TableIndex index;
index.engine_type_ = index_param_.mutable_index()->index_type();
index.nlist_ = index_param_.mutable_index()->nlist();
index.index_file_size_ = index_param_.mutable_index()->index_file_size();
index.metric_type_ = index_param_.mutable_index()->metric_type();
stat = DBWrapper::DB()->CreateIndex(table_name_, index);
if (!stat.ok()) {
@ -904,7 +904,6 @@ DescribeIndexTask::OnExecute() {
index_param_.mutable_table_name()->set_table_name(table_name_);
index_param_.mutable_index()->set_index_type(index.engine_type_);
index_param_.mutable_index()->set_nlist(index.nlist_);
index_param_.mutable_index()->set_index_file_size(index.index_file_size_);
index_param_.mutable_index()->set_metric_type(index.metric_type_);
rc.ElapseFromBegin("totally cost");

2
cpp/src/utils/ValidationUtil.cpp
View File

@ -76,7 +76,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
}
ServerError
ValidationUtil::ValidateTableIndexFileSize(int32_t index_file_size) {
ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
if(index_file_size <= 0 || index_file_size > index_file_size_limit) {
return SERVER_INVALID_INDEX_FILE_SIZE;
}

2
cpp/src/utils/ValidationUtil.h
View File

@ -21,7 +21,7 @@ public:
ValidateTableIndexNlist(int32_t nlist);
static ServerError
ValidateTableIndexFileSize(int32_t index_file_size);
ValidateTableIndexFileSize(int64_t index_file_size);
static ServerError
ValidateTableIndexMetricType(int32_t metric_type);