From 0a2473a567b6e05b9ce9407236fae35e556c10d0 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Thu, 17 Oct 2019 19:33:35 +0800 Subject: [PATCH 01/21] add wrapper unittest Former-commit-id: 8a32c99bb83efe2a0697f038cb210eb18f2f7d6f --- core/src/scheduler/optimizer/Pass.h | 6 ++-- core/unittest/wrapper/test_wrapper.cpp | 11 +++++++ core/unittest/wrapper/utils.h | 44 +++++++++++++------------- 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/core/src/scheduler/optimizer/Pass.h b/core/src/scheduler/optimizer/Pass.h index 959c3ea5ee..1c74c30d69 100644 --- a/core/src/scheduler/optimizer/Pass.h +++ b/core/src/scheduler/optimizer/Pass.h @@ -34,9 +34,9 @@ namespace scheduler { class Pass { public: - virtual void - Init() { - } +// virtual void +// Init() { +// } virtual bool Run(const TaskPtr& task) = 0; diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index e16a4e2bb2..f112fc7e65 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -164,3 +164,14 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { AssertResult(res_ids, res_dis); } } + +#include "wrapper/ConfAdapter.h" +TEST(whatever, test_config) { + milvus::engine::TempMetaConf conf; + auto nsg_conf = std::make_shared(); + nsg_conf->Match(conf); + nsg_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_GPU); + + auto pq_conf = std::make_shared(); + pq_conf->Match(conf); +} diff --git a/core/unittest/wrapper/utils.h b/core/unittest/wrapper/utils.h index b895b599e0..5a614543c9 100644 --- a/core/unittest/wrapper/utils.h +++ b/core/unittest/wrapper/utils.h @@ -104,28 +104,28 @@ class ParamGenerator { tempconf->metric_type = knowhere::METRICTYPE::L2; return tempconf; } - case milvus::engine::IndexType::FAISS_IVFPQ_CPU: - case milvus::engine::IndexType::FAISS_IVFPQ_GPU: { - auto tempconf = std::make_shared(); - tempconf->nlist = 100; - tempconf->nprobe = 16; - tempconf->nbits = 8; - tempconf->m = 8; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; - } - case milvus::engine::IndexType::NSG_MIX: { - auto tempconf = std::make_shared(); - tempconf->nlist = 100; - tempconf->nprobe = 16; - tempconf->search_length = 8; - tempconf->knng = 200; - tempconf->search_length = 40; // TODO(linxj): be 20 when search - tempconf->out_degree = 60; - tempconf->candidate_pool_size = 200; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; - } +// case milvus::engine::IndexType::FAISS_IVFPQ_CPU: +// case milvus::engine::IndexType::FAISS_IVFPQ_GPU: { +// auto tempconf = std::make_shared(); +// tempconf->nlist = 100; +// tempconf->nprobe = 16; +// tempconf->nbits = 8; +// tempconf->m = 8; +// tempconf->metric_type = knowhere::METRICTYPE::L2; +// return tempconf; +// } +// case milvus::engine::IndexType::NSG_MIX: { +// auto tempconf = std::make_shared(); +// tempconf->nlist = 100; +// tempconf->nprobe = 16; +// tempconf->search_length = 8; +// tempconf->knng = 200; +// tempconf->search_length = 40; // TODO(linxj): be 20 when search +// tempconf->out_degree = 60; +// tempconf->candidate_pool_size = 200; +// tempconf->metric_type = knowhere::METRICTYPE::L2; +// return tempconf; +// } } } }; From 7a10d01fa1491c1fa4f4126607cbdcff52f586f6 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Thu, 17 Oct 2019 19:33:35 +0800 Subject: [PATCH 02/21] add wrapper unittest Former-commit-id: b4f68eeaf55e4960f25239e983c3ae75e5e5ed40 --- core/src/scheduler/optimizer/Pass.h | 6 ++-- core/unittest/wrapper/test_wrapper.cpp | 11 +++++++ core/unittest/wrapper/utils.h | 44 +++++++++++++------------- 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/core/src/scheduler/optimizer/Pass.h b/core/src/scheduler/optimizer/Pass.h index 959c3ea5ee..1c74c30d69 100644 --- a/core/src/scheduler/optimizer/Pass.h +++ b/core/src/scheduler/optimizer/Pass.h @@ -34,9 +34,9 @@ namespace scheduler { class Pass { public: - virtual void - Init() { - } +// virtual void +// Init() { +// } virtual bool Run(const TaskPtr& task) = 0; diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index e16a4e2bb2..f112fc7e65 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -164,3 +164,14 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { AssertResult(res_ids, res_dis); } } + +#include "wrapper/ConfAdapter.h" +TEST(whatever, test_config) { + milvus::engine::TempMetaConf conf; + auto nsg_conf = std::make_shared(); + nsg_conf->Match(conf); + nsg_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_GPU); + + auto pq_conf = std::make_shared(); + pq_conf->Match(conf); +} diff --git a/core/unittest/wrapper/utils.h b/core/unittest/wrapper/utils.h index b895b599e0..5a614543c9 100644 --- a/core/unittest/wrapper/utils.h +++ b/core/unittest/wrapper/utils.h @@ -104,28 +104,28 @@ class ParamGenerator { tempconf->metric_type = knowhere::METRICTYPE::L2; return tempconf; } - case milvus::engine::IndexType::FAISS_IVFPQ_CPU: - case milvus::engine::IndexType::FAISS_IVFPQ_GPU: { - auto tempconf = std::make_shared(); - tempconf->nlist = 100; - tempconf->nprobe = 16; - tempconf->nbits = 8; - tempconf->m = 8; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; - } - case milvus::engine::IndexType::NSG_MIX: { - auto tempconf = std::make_shared(); - tempconf->nlist = 100; - tempconf->nprobe = 16; - tempconf->search_length = 8; - tempconf->knng = 200; - tempconf->search_length = 40; // TODO(linxj): be 20 when search - tempconf->out_degree = 60; - tempconf->candidate_pool_size = 200; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; - } +// case milvus::engine::IndexType::FAISS_IVFPQ_CPU: +// case milvus::engine::IndexType::FAISS_IVFPQ_GPU: { +// auto tempconf = std::make_shared(); +// tempconf->nlist = 100; +// tempconf->nprobe = 16; +// tempconf->nbits = 8; +// tempconf->m = 8; +// tempconf->metric_type = knowhere::METRICTYPE::L2; +// return tempconf; +// } +// case milvus::engine::IndexType::NSG_MIX: { +// auto tempconf = std::make_shared(); +// tempconf->nlist = 100; +// tempconf->nprobe = 16; +// tempconf->search_length = 8; +// tempconf->knng = 200; +// tempconf->search_length = 40; // TODO(linxj): be 20 when search +// tempconf->out_degree = 60; +// tempconf->candidate_pool_size = 200; +// tempconf->metric_type = knowhere::METRICTYPE::L2; +// return tempconf; +// } } } }; From 1f7dad0497e4e997857e41a859d0606886b15bed Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Thu, 17 Oct 2019 19:38:03 +0800 Subject: [PATCH 03/21] fix clang format Former-commit-id: b158a9b2a85a182882a666c068601f7dd4ef1f23 --- core/src/scheduler/optimizer/Pass.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/scheduler/optimizer/Pass.h b/core/src/scheduler/optimizer/Pass.h index 1c74c30d69..016b05e457 100644 --- a/core/src/scheduler/optimizer/Pass.h +++ b/core/src/scheduler/optimizer/Pass.h @@ -34,9 +34,9 @@ namespace scheduler { class Pass { public: -// virtual void -// Init() { -// } + // virtual void + // Init() { + // } virtual bool Run(const TaskPtr& task) = 0; From 938de62fa171aceb5e8e7542cb879d49e35392df Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 18 Oct 2019 11:46:15 +0800 Subject: [PATCH 04/21] #39 Create SQ8H index hang if using github server version Former-commit-id: 056ccbf9cad2de039cc61a512932d3aaea25131d --- CHANGELOG.md | 1 + core/src/db/engine/ExecutionEngineImpl.cpp | 6 +++--- core/src/scheduler/task/BuildIndexTask.cpp | 6 ++++++ core/src/utils/ValidationUtil.cpp | 9 +++++++++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1119c8200e..f2e22d0112 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#20 - C++ sdk example get grpc error - \#23 - Add unittest to improve code coverage - \#31 - make clang-format failed after run build.sh -l +- \#39 - Create SQ8H index hang if using github server version ## Improvement - MS-552 - Add and change the easylogging library diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index bde45cf112..df802c27e7 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -65,7 +65,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& : location_(location), dim_(dimension), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) { index_ = CreatetVecIndex(EngineType::FAISS_IDMAP); if (!index_) { - throw Exception(DB_ERROR, "Could not create VecIndex"); + throw Exception(DB_ERROR, "Unsupported index type"); } TempMetaConf temp_conf; @@ -111,7 +111,7 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { break; } default: { - ENGINE_LOG_ERROR << "Invalid engine type"; + ENGINE_LOG_ERROR << "Unsupported index type"; return nullptr; } } @@ -373,7 +373,7 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t auto to_index = CreatetVecIndex(engine_type); if (!to_index) { - throw Exception(DB_ERROR, "Could not create VecIndex"); + throw Exception(DB_ERROR, "Unsupported index type"); } TempMetaConf temp_conf; diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index 26a2a44efc..d8602c141e 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -134,6 +134,7 @@ XBuildIndexTask::Execute() { ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; + build_index_job->BuildIndexDone(to_index_id_); to_index_engine_ = nullptr; return; } @@ -148,6 +149,7 @@ XBuildIndexTask::Execute() { std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" << std::endl; + build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); to_index_engine_ = nullptr; return; @@ -158,6 +160,9 @@ XBuildIndexTask::Execute() { meta_ptr->HasTable(file_->table_id_, has_table); if (!has_table) { meta_ptr->DeleteTableFiles(file_->table_id_); + + build_index_job->BuildIndexDone(to_index_id_); + build_index_job->GetStatus() = Status(DB_ERROR, "Table has been deleted, discard index file."); to_index_engine_ = nullptr; return; } @@ -177,6 +182,7 @@ XBuildIndexTask::Execute() { std::cout << "ERROR: failed to persist index file: " << table_file.location_ << ", possible out of disk space" << std::endl; + build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); to_index_engine_ = nullptr; return; diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index b982a31f5e..e275f533a3 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -94,6 +94,15 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) { return Status(SERVER_INVALID_INDEX_TYPE, msg); } +#ifndef CUSTOMIZATION + // special case, hybird index only available in customize faiss library + if (engine_type == static_cast(engine::EngineType::FAISS_IVFSQ8H)) { + std::string msg = "Unsupported index type: " + std::to_string(index_type); + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_INDEX_TYPE, msg); + } +#endif + return Status::OK(); } From 92a21ec3dab5b8c8ee50548808fc0bcc6103e6a5 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 18 Oct 2019 14:40:15 +0800 Subject: [PATCH 05/21] #30 Some troubleshoot messages in Milvus do not provide enough information Former-commit-id: 49b03511fc7abbc323c511fad28a568d71b518d5 --- CHANGELOG.md | 1 + core/src/server/grpc_impl/GrpcRequestTask.cpp | 40 ++++++++++--------- core/src/utils/ValidationUtil.cpp | 38 ++++++++++-------- 3 files changed, 44 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2e22d0112..257011cc03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#23 - Add unittest to improve code coverage - \#31 - make clang-format failed after run build.sh -l - \#39 - Create SQ8H index hang if using github server version +- \#30 - Some troubleshoot messages in Milvus do not provide enough information ## Improvement - MS-552 - Add and change the easylogging library diff --git a/core/src/server/grpc_impl/GrpcRequestTask.cpp b/core/src/server/grpc_impl/GrpcRequestTask.cpp index 02cb24175a..9e14ecece8 100644 --- a/core/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/core/src/server/grpc_impl/GrpcRequestTask.cpp @@ -113,6 +113,12 @@ ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, return Status::OK(); } + +std::string TableNotExistMsg(const std::string& table_name) { + return "Table " + table_name + + " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name exists."; +} + } // namespace //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -255,7 +261,7 @@ CreateIndexTask::OnExecute() { } if (!has_table) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); } auto& grpc_index = index_param_->index(); @@ -348,7 +354,7 @@ DropTableTask::OnExecute() { status = DBWrapper::DB()->DescribeTable(table_info); if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); } else { return status; } @@ -420,12 +426,12 @@ InsertTask::OnExecute() { return status; } if (insert_param_->row_record_array().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, "The vector array is empty. Make sure you have entered vector records."); } if (!insert_param_->row_id_array().empty()) { if (insert_param_->row_id_array().size() != insert_param_->row_record_array_size()) { - return Status(SERVER_ILLEGAL_VECTOR_ID, "Size of vector ids is not equal to row record array size"); + return Status(SERVER_ILLEGAL_VECTOR_ID, "The size of vector ID array must be equal to the size of the vector."); } } @@ -435,7 +441,7 @@ InsertTask::OnExecute() { status = DBWrapper::DB()->DescribeTable(table_info); if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + insert_param_->table_name() + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(insert_param_->table_name())); } else { return status; } @@ -447,13 +453,13 @@ InsertTask::OnExecute() { // user already provided id before, all insert action require user id if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) { return Status(SERVER_ILLEGAL_VECTOR_ID, - "Table vector ids are user defined, please provide id for this batch"); + "Table vector IDs are user-defined. Please provide IDs for all vectors of this table."); } // user didn't provided id before, no need to provide user id if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) { return Status(SERVER_ILLEGAL_VECTOR_ID, - "Table vector ids are auto generated, no need to provide id for this batch"); + "Table vector IDs are auto-generated. All vectors of this table must use auto-generated IDs."); } rc.RecordSection("check validation"); @@ -470,13 +476,12 @@ InsertTask::OnExecute() { // TODO(yk): change to one dimension array or use multiple-thread to copy the data for (size_t i = 0; i < insert_param_->row_record_array_size(); i++) { if (insert_param_->row_record_array(i).vector_data().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array data is empty"); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, "The vector dimension must be equal to the table dimension."); } uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); if (vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "Invalid row record dimension: " + std::to_string(vec_dim) + - " vs. table dimension:" + std::to_string(table_info.dimension_); + std::string error_msg = "The vector dimension must be equal to the table dimension."; return Status(error_code, error_msg); } memcpy(&vec_f[i * table_info.dimension_], insert_param_->row_record_array(i).vector_data().data(), @@ -569,7 +574,7 @@ SearchTask::OnExecute() { status = DBWrapper::DB()->DescribeTable(table_info); if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); } else { return status; } @@ -587,7 +592,7 @@ SearchTask::OnExecute() { } if (search_param_->query_record_array().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, "The vector array is empty. Make sure you have entered vector records."); } // step 4: check date range, and convert to db dates @@ -609,13 +614,12 @@ SearchTask::OnExecute() { std::vector vec_f(record_array_size * table_info.dimension_, 0); for (size_t i = 0; i < record_array_size; i++) { if (search_param_->query_record_array(i).vector_data().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array data is empty"); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, "The vector dimension must be equal to the table dimension."); } uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); if (query_vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "Invalid row record dimension: " + std::to_string(query_vec_dim) + - " vs. table dimension:" + std::to_string(table_info.dimension_); + std::string error_msg = "The vector dimension must be equal to the table dimension."; return Status(error_code, error_msg); } @@ -707,7 +711,7 @@ CountTableTask::OnExecute() { status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); if (!status.ok()) { if (status.code(), DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); } else { return status; } @@ -779,7 +783,7 @@ DeleteByRangeTask::OnExecute() { status = DBWrapper::DB()->DescribeTable(table_info); if (!status.ok()) { if (status.code(), DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name)); } else { return status; } @@ -917,7 +921,7 @@ DropIndexTask::OnExecute() { } if (!has_table) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); } // step 2: check table existence diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index e275f533a3..11ff5da68e 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -37,14 +37,15 @@ Status ValidationUtil::ValidateTableName(const std::string& table_name) { // Table name shouldn't be empty. if (table_name.empty()) { - std::string msg = "Empty table name"; + std::string msg = "Table name should not be empty."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_TABLE_NAME, msg); } + std::string invalid_msg = "Invalid table name: " + table_name + ". "; // Table name size shouldn't exceed 16384. if (table_name.size() > TABLE_NAME_SIZE_LIMIT) { - std::string msg = "Table name size exceed the limitation"; + std::string msg = invalid_msg + "The length of a table name must be less than 255 characters."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_TABLE_NAME, msg); } @@ -52,7 +53,7 @@ ValidationUtil::ValidateTableName(const std::string& table_name) { // Table name first character should be underscore or character. char first_char = table_name[0]; if (first_char != '_' && std::isalpha(first_char) == 0) { - std::string msg = "Table name first character isn't underscore or character"; + std::string msg = invalid_msg + "The first character of a table name must be an underscore or letter."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_TABLE_NAME, msg); } @@ -61,7 +62,7 @@ ValidationUtil::ValidateTableName(const std::string& table_name) { for (int64_t i = 1; i < table_name_size; ++i) { char name_char = table_name[i]; if (name_char != '_' && std::isalnum(name_char) == 0) { - std::string msg = "Table name character isn't underscore or alphanumber"; + std::string msg = invalid_msg + "Table name can only contain numbers, letters, and underscores."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_TABLE_NAME, msg); } @@ -72,12 +73,9 @@ ValidationUtil::ValidateTableName(const std::string& table_name) { Status ValidationUtil::ValidateTableDimension(int64_t dimension) { - if (dimension <= 0) { - std::string msg = "Dimension value should be greater than 0"; - SERVER_LOG_ERROR << msg; - return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); - } else if (dimension > TABLE_DIMENSION_LIMIT) { - std::string msg = "Table dimension excceed the limitation: " + std::to_string(TABLE_DIMENSION_LIMIT); + if (dimension <= 0 || dimension > TABLE_DIMENSION_LIMIT) { + std::string msg = "Invalid table dimension: " + std::to_string(dimension) + ". " + + "The table dimension must be within the range of 1 ~ 16384."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); } else { @@ -89,7 +87,8 @@ Status ValidationUtil::ValidateTableIndexType(int32_t index_type) { int engine_type = static_cast(engine::EngineType(index_type)); if (engine_type <= 0 || engine_type > static_cast(engine::EngineType::MAX_VALUE)) { - std::string msg = "Invalid index type: " + std::to_string(index_type); + std::string msg = "Invalid index type: " + std::to_string(index_type) + ". " + + "Make sure the index type is in IndexType list."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_TYPE, msg); } @@ -109,7 +108,8 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) { Status ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { if (nlist <= 0) { - std::string msg = "nlist value should be greater than 0"; + std::string msg = "Invalid index nlist: " + std::to_string(nlist) + ". " + + "The index nlist must be greater than 0."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_NLIST, msg); } @@ -120,7 +120,9 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { Status ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) { - std::string msg = "Invalid index file size: " + std::to_string(index_file_size); + std::string msg = "Invalid index file size: " + std::to_string(index_file_size) + ". " + + "The index file size must be within the range of 1 ~ " + + std::to_string(INDEX_FILE_SIZE_LIMIT) + "."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_FILE_SIZE, msg); } @@ -132,7 +134,8 @@ Status ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { if (metric_type != static_cast(engine::MetricType::L2) && metric_type != static_cast(engine::MetricType::IP)) { - std::string msg = "Invalid metric type: " + std::to_string(metric_type); + std::string msg = "Invalid index metric type: " + std::to_string(metric_type) + ". " + + "Make sure the metric type is either MetricType.L2 or MetricType.IP."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_METRIC_TYPE, msg); } @@ -142,7 +145,8 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { Status ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) { if (top_k <= 0 || top_k > 2048) { - std::string msg = "Invalid top k value: " + std::to_string(top_k) + ", rational range [1, 2048]"; + std::string msg = "Invalid topk: " + std::to_string(top_k) + ". " + + "The topk must be within the range of 1 ~ 2048."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_TOPK, msg); } @@ -153,8 +157,8 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem Status ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) { if (nprobe <= 0 || nprobe > table_schema.nlist_) { - std::string msg = "Invalid nprobe value: " + std::to_string(nprobe) + ", rational range [1, " + - std::to_string(table_schema.nlist_) + "]"; + std::string msg = "Invalid nprobe: " + std::to_string(nprobe) + ". " + + "The nprobe must be within the range of 1 ~ index nlist."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_NPROBE, msg); } From bcc4bbde0365c1cfbee828ea7dedbaf660aa86c5 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 18 Oct 2019 14:41:32 +0800 Subject: [PATCH 06/21] format code Former-commit-id: 993b88487da976f809fdb29ffb07cdb0188a16f7 --- core/src/server/grpc_impl/GrpcRequestTask.cpp | 28 ++++++++++------- core/src/utils/ValidationUtil.cpp | 30 +++++++++---------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/core/src/server/grpc_impl/GrpcRequestTask.cpp b/core/src/server/grpc_impl/GrpcRequestTask.cpp index 9e14ecece8..86ff23b3d0 100644 --- a/core/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/core/src/server/grpc_impl/GrpcRequestTask.cpp @@ -114,9 +114,11 @@ ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, return Status::OK(); } -std::string TableNotExistMsg(const std::string& table_name) { - return "Table " + table_name - + " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name exists."; +std::string +TableNotExistMsg(const std::string& table_name) { + return "Table " + table_name + + " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name " + "exists."; } } // namespace @@ -426,12 +428,14 @@ InsertTask::OnExecute() { return status; } if (insert_param_->row_record_array().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, "The vector array is empty. Make sure you have entered vector records."); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector array is empty. Make sure you have entered vector records."); } if (!insert_param_->row_id_array().empty()) { if (insert_param_->row_id_array().size() != insert_param_->row_record_array_size()) { - return Status(SERVER_ILLEGAL_VECTOR_ID, "The size of vector ID array must be equal to the size of the vector."); + return Status(SERVER_ILLEGAL_VECTOR_ID, + "The size of vector ID array must be equal to the size of the vector."); } } @@ -458,8 +462,9 @@ InsertTask::OnExecute() { // user didn't provided id before, no need to provide user id if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) { - return Status(SERVER_ILLEGAL_VECTOR_ID, - "Table vector IDs are auto-generated. All vectors of this table must use auto-generated IDs."); + return Status( + SERVER_ILLEGAL_VECTOR_ID, + "Table vector IDs are auto-generated. All vectors of this table must use auto-generated IDs."); } rc.RecordSection("check validation"); @@ -476,7 +481,8 @@ InsertTask::OnExecute() { // TODO(yk): change to one dimension array or use multiple-thread to copy the data for (size_t i = 0; i < insert_param_->row_record_array_size(); i++) { if (insert_param_->row_record_array(i).vector_data().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, "The vector dimension must be equal to the table dimension."); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector dimension must be equal to the table dimension."); } uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); if (vec_dim != table_info.dimension_) { @@ -592,7 +598,8 @@ SearchTask::OnExecute() { } if (search_param_->query_record_array().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, "The vector array is empty. Make sure you have entered vector records."); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector array is empty. Make sure you have entered vector records."); } // step 4: check date range, and convert to db dates @@ -614,7 +621,8 @@ SearchTask::OnExecute() { std::vector vec_f(record_array_size * table_info.dimension_, 0); for (size_t i = 0; i < record_array_size; i++) { if (search_param_->query_record_array(i).vector_data().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, "The vector dimension must be equal to the table dimension."); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector dimension must be equal to the table dimension."); } uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); if (query_vec_dim != table_info.dimension_) { diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index 11ff5da68e..68088d6c93 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -74,8 +74,8 @@ ValidationUtil::ValidateTableName(const std::string& table_name) { Status ValidationUtil::ValidateTableDimension(int64_t dimension) { if (dimension <= 0 || dimension > TABLE_DIMENSION_LIMIT) { - std::string msg = "Invalid table dimension: " + std::to_string(dimension) + ". " - + "The table dimension must be within the range of 1 ~ 16384."; + std::string msg = "Invalid table dimension: " + std::to_string(dimension) + ". " + + "The table dimension must be within the range of 1 ~ 16384."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); } else { @@ -87,8 +87,8 @@ Status ValidationUtil::ValidateTableIndexType(int32_t index_type) { int engine_type = static_cast(engine::EngineType(index_type)); if (engine_type <= 0 || engine_type > static_cast(engine::EngineType::MAX_VALUE)) { - std::string msg = "Invalid index type: " + std::to_string(index_type) + ". " - + "Make sure the index type is in IndexType list."; + std::string msg = "Invalid index type: " + std::to_string(index_type) + ". " + + "Make sure the index type is in IndexType list."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_TYPE, msg); } @@ -108,8 +108,8 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) { Status ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { if (nlist <= 0) { - std::string msg = "Invalid index nlist: " + std::to_string(nlist) + ". " - + "The index nlist must be greater than 0."; + std::string msg = + "Invalid index nlist: " + std::to_string(nlist) + ". " + "The index nlist must be greater than 0."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_NLIST, msg); } @@ -120,9 +120,9 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { Status ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) { - std::string msg = "Invalid index file size: " + std::to_string(index_file_size) + ". " - + "The index file size must be within the range of 1 ~ " - + std::to_string(INDEX_FILE_SIZE_LIMIT) + "."; + std::string msg = "Invalid index file size: " + std::to_string(index_file_size) + ". " + + "The index file size must be within the range of 1 ~ " + + std::to_string(INDEX_FILE_SIZE_LIMIT) + "."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_FILE_SIZE, msg); } @@ -134,8 +134,8 @@ Status ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { if (metric_type != static_cast(engine::MetricType::L2) && metric_type != static_cast(engine::MetricType::IP)) { - std::string msg = "Invalid index metric type: " + std::to_string(metric_type) + ". " - + "Make sure the metric type is either MetricType.L2 or MetricType.IP."; + std::string msg = "Invalid index metric type: " + std::to_string(metric_type) + ". " + + "Make sure the metric type is either MetricType.L2 or MetricType.IP."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_METRIC_TYPE, msg); } @@ -145,8 +145,8 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { Status ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) { if (top_k <= 0 || top_k > 2048) { - std::string msg = "Invalid topk: " + std::to_string(top_k) + ". " - + "The topk must be within the range of 1 ~ 2048."; + std::string msg = + "Invalid topk: " + std::to_string(top_k) + ". " + "The topk must be within the range of 1 ~ 2048."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_TOPK, msg); } @@ -157,8 +157,8 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem Status ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) { if (nprobe <= 0 || nprobe > table_schema.nlist_) { - std::string msg = "Invalid nprobe: " + std::to_string(nprobe) + ". " - + "The nprobe must be within the range of 1 ~ index nlist."; + std::string msg = "Invalid nprobe: " + std::to_string(nprobe) + ". " + + "The nprobe must be within the range of 1 ~ index nlist."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_NPROBE, msg); } From 4d1fb143e1339dbc015a02642ac1b5aa47d0fbef Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Fri, 18 Oct 2019 16:32:28 +0800 Subject: [PATCH 07/21] MS-665 rename build_index_gpu to index_build_device, and move it to resource config Former-commit-id: f3da7963531406da7a22fbf5ab95caa8e920344a --- CHANGELOG.md | 1 + core/conf/server_config.template | 5 +- core/src/db/engine/ExecutionEngineImpl.cpp | 2 +- core/src/scheduler/SchedInst.cpp | 2 +- core/src/scheduler/Utils.cpp | 2 +- .../scheduler/action/PushTaskToNeighbour.cpp | 2 +- core/src/server/Config.cpp | 128 ++++++++++-------- core/src/server/Config.h | 22 +-- core/src/wrapper/KnowhereResource.cpp | 4 +- .../server/appendix/server_config.yaml | 26 ++-- 10 files changed, 109 insertions(+), 85 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 257011cc03..434a91846e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-653 - When config check fail, Milvus close without message - MS-654 - Describe index timeout when building index - MS-658 - Fix SQ8 Hybrid can't search +- MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources - \#20 - C++ sdk example get grpc error - \#23 - Add unittest to improve code coverage - \#31 - make clang-format failed after run build.sh -l diff --git a/core/conf/server_config.template b/core/conf/server_config.template index 27c2b12cfe..7abfb8b055 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_config.template @@ -16,7 +16,6 @@ db_config: insert_buffer_size: 4 # GB, maximum insert buffer size allowed # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory - build_index_gpu: 0 # gpu id used for building index preload_table: # preload data at startup, '*' means load all tables, empty value means no preload # you can specify preload tables like this: table1,table2,table3 @@ -39,6 +38,6 @@ engine_config: # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times resource_config: - resource_pool: - - cpu + search_resources: # define the GPUs used for search computation, valid value: gpux - gpu0 + index_build_device: gpu0 # GPU used for building index \ No newline at end of file diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index df802c27e7..862c1026d2 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -503,7 +503,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); - Status s = config.GetDBConfigBuildIndexGPU(gpu_num_); + Status s = config.GetResourceConfigIndexBuildDevice(gpu_num_); if (!s.ok()) { return s; } diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 0053332746..4d8fadd38d 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -50,7 +50,7 @@ load_simple_config() { std::string mode; config.GetResourceConfigMode(mode); std::vector pool; - config.GetResourceConfigPool(pool); + config.GetResourceConfigSearchResources(pool); // get resources bool use_cpu_to_compute = false; diff --git a/core/src/scheduler/Utils.cpp b/core/src/scheduler/Utils.cpp index 18f6fc249d..998e545ba5 100644 --- a/core/src/scheduler/Utils.cpp +++ b/core/src/scheduler/Utils.cpp @@ -48,7 +48,7 @@ get_gpu_pool() { server::Config& config = server::Config::GetInstance(); std::vector pool; - Status s = config.GetResourceConfigPool(pool); + Status s = config.GetResourceConfigSearchResources(pool); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } diff --git a/core/src/scheduler/action/PushTaskToNeighbour.cpp b/core/src/scheduler/action/PushTaskToNeighbour.cpp index 4e7dbf984f..4c8155342b 100644 --- a/core/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/core/src/scheduler/action/PushTaskToNeighbour.cpp @@ -184,7 +184,7 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr // get build index gpu resource server::Config& config = server::Config::GetInstance(); int32_t build_index_gpu; - Status stat = config.GetDBConfigBuildIndexGPU(build_index_gpu); + Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); bool find_gpu_res = false; for (uint64_t i = 0; i < compute_resources.size(); ++i) { diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 189070eb2b..7de84cbccc 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -129,12 +130,6 @@ Config::ValidateConfig() { return s; } - int32_t db_build_index_gpu; - s = GetDBConfigBuildIndexGPU(db_build_index_gpu); - if (!s.ok()) { - return s; - } - /* metric config */ bool metric_enable_monitor; s = GetMetricConfigEnableMonitor(metric_enable_monitor); @@ -205,8 +200,14 @@ Config::ValidateConfig() { return s; } - std::vector resource_pool; - s = GetResourceConfigPool(resource_pool); + std::vector search_resources; + s = GetResourceConfigSearchResources(search_resources); + if (!s.ok()) { + return s; + } + + int32_t resource_index_build_device; + s = GetResourceConfigIndexBuildDevice(resource_index_build_device); if (!s.ok()) { return s; } @@ -270,11 +271,6 @@ Config::ResetDefaultConfig() { return s; } - s = SetDBConfigBuildIndexGPU(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - if (!s.ok()) { - return s; - } - /* metric config */ s = SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); if (!s.ok()) { @@ -334,6 +330,11 @@ Config::ResetDefaultConfig() { return s; } + s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + if (!s.ok()) { + return s; + } + return Status::OK(); } @@ -459,19 +460,6 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { return Status::OK(); } -Status -Config::CheckDBConfigBuildIndexGPU(const std::string& value) { - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value); - } else { - int32_t gpu_index = std::stoi(value); - if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value); - } - } - return Status::OK(); -} - Status Config::CheckMetricConfigEnableMonitor(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { @@ -544,7 +532,7 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { } else { uint64_t gpu_cache_capacity = std::stoi(value) * GB; int gpu_index; - Status s = GetDBConfigBuildIndexGPU(gpu_index); + Status s = GetResourceConfigIndexBuildDevice(gpu_index); if (!s.ok()) { return s; } @@ -616,9 +604,38 @@ Config::CheckResourceConfigMode(const std::string& value) { } Status -Config::CheckResourceConfigPool(const std::vector& value) { +CheckGpuDevice(const std::string& value) { + const std::regex pat("gpu(\\d+)"); + std::cmatch m; + if (!std::regex_match(value.c_str(), m, pat)) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid gpu device: " + value); + } + + int32_t gpu_index = std::stoi(value.substr(3)); + if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid gpu device: " + value); + } + return Status::OK(); +} + +Status +Config::CheckResourceConfigSearchResources(const std::vector& value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config pool"); + return Status(SERVER_INVALID_ARGUMENT, "Empty resource config search_resources"); + } + + for (auto& gpu_device : value) { + if (!CheckGpuDevice(gpu_device).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config search_resources: " + gpu_device); + } + } + return Status::OK(); +} + +Status +Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { + if (!CheckGpuDevice(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config index_build_device: " + value); } return Status::OK(); } @@ -739,18 +756,6 @@ Config::GetDBConfigInsertBufferSize(int32_t& value) { return Status::OK(); } -Status -Config::GetDBConfigBuildIndexGPU(int32_t& value) { - std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - Status s = CheckDBConfigBuildIndexGPU(str); - if (!s.ok()) { - return s; - } - - value = std::stoi(str); - return Status::OK(); -} - Status Config::GetDBConfigPreloadTable(std::string& value) { value = GetConfigStr(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE); @@ -880,10 +885,23 @@ Config::GetResourceConfigMode(std::string& value) { } Status -Config::GetResourceConfigPool(std::vector& value) { +Config::GetResourceConfigSearchResources(std::vector& value) { ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE); - value = resource_config.GetSequence(CONFIG_RESOURCE_POOL); - return CheckResourceConfigPool(value); + value = resource_config.GetSequence(CONFIG_RESOURCE_SEARCH_RESOURCES); + return CheckResourceConfigSearchResources(value); +} + +Status +Config::GetResourceConfigIndexBuildDevice(int32_t& value) { + std::string str = + GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + Status s = CheckResourceConfigIndexBuildDevice(str); + if (!s.ok()) { + return s; + } + + value = std::stoi(str.substr(3)); + return Status::OK(); } /////////////////////////////////////////////////////////////////////////////// @@ -999,17 +1017,6 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) { return Status::OK(); } -Status -Config::SetDBConfigBuildIndexGPU(const std::string& value) { - Status s = CheckDBConfigBuildIndexGPU(value); - if (!s.ok()) { - return s; - } - - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); - return Status::OK(); -} - /* metric config */ Status Config::SetMetricConfigEnableMonitor(const std::string& value) { @@ -1135,5 +1142,16 @@ Config::SetResourceConfigMode(const std::string& value) { return Status::OK(); } +Status +Config::SetResourceConfigIndexBuildDevice(const std::string& value) { + Status s = CheckResourceConfigIndexBuildDevice(value); + if (!s.ok()) { + return s; + } + + SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value); + return Status::OK(); +} + } // namespace server } // namespace milvus diff --git a/core/src/server/Config.h b/core/src/server/Config.h index f40c9f2224..4cab25a1c6 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -53,8 +53,6 @@ static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; -static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; -static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; /* cache config */ @@ -91,7 +89,9 @@ static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; static const char* CONFIG_RESOURCE = "resource_config"; static const char* CONFIG_RESOURCE_MODE = "mode"; static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char* CONFIG_RESOURCE_POOL = "resource_pool"; +static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0"; class Config { public: @@ -140,8 +140,6 @@ class Config { CheckDBConfigArchiveDaysThreshold(const std::string& value); Status CheckDBConfigInsertBufferSize(const std::string& value); - Status - CheckDBConfigBuildIndexGPU(const std::string& value); /* metric config */ Status @@ -173,7 +171,9 @@ class Config { Status CheckResourceConfigMode(const std::string& value); Status - CheckResourceConfigPool(const std::vector& value); + CheckResourceConfigSearchResources(const std::vector& value); + Status + CheckResourceConfigIndexBuildDevice(const std::string& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -203,8 +203,6 @@ class Config { Status GetDBConfigInsertBufferSize(int32_t& value); Status - GetDBConfigBuildIndexGPU(int32_t& value); - Status GetDBConfigPreloadTable(std::string& value); /* metric config */ @@ -237,7 +235,9 @@ class Config { Status GetResourceConfigMode(std::string& value); Status - GetResourceConfigPool(std::vector& value); + GetResourceConfigSearchResources(std::vector& value); + Status + GetResourceConfigIndexBuildDevice(int32_t& value); public: /* server config */ @@ -263,8 +263,6 @@ class Config { SetDBConfigArchiveDaysThreshold(const std::string& value); Status SetDBConfigInsertBufferSize(const std::string& value); - Status - SetDBConfigBuildIndexGPU(const std::string& value); /* metric config */ Status @@ -295,6 +293,8 @@ class Config { /* resource config */ Status SetResourceConfigMode(const std::string& value); + Status + SetResourceConfigIndexBuildDevice(const std::string& value); private: std::unordered_map> config_map_; diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index d291bb9299..650ae727c1 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -45,7 +45,7 @@ KnowhereResource::Initialize() { server::Config& config = server::Config::GetInstance(); int32_t build_index_gpu; - s = config.GetDBConfigBuildIndexGPU(build_index_gpu); + s = config.GetResourceConfigIndexBuildDevice(build_index_gpu); if (!s.ok()) return s; @@ -53,7 +53,7 @@ KnowhereResource::Initialize() { // get search gpu resource std::vector pool; - s = config.GetResourceConfigPool(pool); + s = config.GetResourceConfigSearchResources(pool); if (!s.ok()) return s; diff --git a/core/unittest/server/appendix/server_config.yaml b/core/unittest/server/appendix/server_config.yaml index f92b2f1a18..c5aef82511 100644 --- a/core/unittest/server/appendix/server_config.yaml +++ b/core/unittest/server/appendix/server_config.yaml @@ -1,4 +1,4 @@ -# All the following configurations are default values. +# Default values are used when you make no changes to the following parameters. server_config: address: 0.0.0.0 # milvus server ip address (IPv4) @@ -11,27 +11,33 @@ db_config: secondary_path: # path used to store data only, split by semicolon backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database - # Keep 'dialect://:@:/', and replace other texts with real values. + # Keep 'dialect://:@:/', and replace other texts with real values # Replace 'dialect' with 'mysql' or 'sqlite' insert_buffer_size: 4 # GB, maximum insert buffer size allowed - build_index_gpu: 0 # gpu id used for building index + # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory + + preload_table: # preload data at startup, '*' means load all tables, empty value means no preload + # you can specify preload tables like this: table1,table2,table3 metric_config: enable_monitor: false # enable monitoring or not collector: prometheus # prometheus prometheus_config: - port: 8080 # port prometheus used to fetch metrics + port: 8080 # port prometheus uses to fetch metrics cache_config: - cpu_mem_capacity: 16 # GB, CPU memory used for cache - cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered - cache_insert_data: false # whether load inserted data into cache + cpu_cache_capacity: 16 # GB, CPU memory used for cache + cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + gpu_cache_capacity: 4 # GB, GPU memory used for cache + gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + cache_insert_data: false # whether to load inserted data into cache engine_config: - blas_threshold: 20 + use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times + # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times resource_config: - resource_pool: - - cpu + search_resources: # define the GPUs used for search computation, valid value: gpux - gpu0 + index_build_device: gpu0 # GPU used for building index From 98e5830e14d7d87f8f53bf689ed7cd5ecc5d3e35 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Fri, 18 Oct 2019 17:45:29 +0800 Subject: [PATCH 08/21] Put union of index_build_device and search resources to gpu_pool Former-commit-id: 8e8f3b081c3af8152cddb1c250a1261f475c7f8a --- core/src/scheduler/SchedInst.cpp | 25 +++++++++++++------ .../scheduler/action/PushTaskToNeighbour.cpp | 4 +-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 4d8fadd38d..1bd1ca00cf 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -53,26 +53,35 @@ load_simple_config() { config.GetResourceConfigSearchResources(pool); // get resources - bool use_cpu_to_compute = false; - for (auto& resource : pool) { - if (resource == "cpu") { - use_cpu_to_compute = true; - break; - } - } auto gpu_ids = get_gpu_pool(); + int32_t build_gpu_id; + config.GetResourceConfigIndexBuildDevice(build_gpu_id); + // create and connect ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false)); auto io = Connection("io", 500); - ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, use_cpu_to_compute)); + ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true)); ResMgrInst::GetInstance()->Connect("disk", "cpu", io); auto pcie = Connection("pcie", 12000); + bool find_build_gpu_id = false; for (auto& gpu_id : gpu_ids) { ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); + if (build_gpu_id == gpu_id) { + find_build_gpu_id = true; + } + } + + if (not find_build_gpu_id) { + ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(build_gpu_id), + "GPU", + build_gpu_id, + true, + true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(build_gpu_id), pcie); } } diff --git a/core/src/scheduler/action/PushTaskToNeighbour.cpp b/core/src/scheduler/action/PushTaskToNeighbour.cpp index 4c8155342b..c64e81dcfa 100644 --- a/core/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/core/src/scheduler/action/PushTaskToNeighbour.cpp @@ -187,8 +187,8 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); bool find_gpu_res = false; - for (uint64_t i = 0; i < compute_resources.size(); ++i) { - if (res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { + if (res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { + for (uint64_t i = 0; i < compute_resources.size(); ++i) { if (compute_resources[i]->name() == res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu)->name()) { find_gpu_res = true; From f29bad6146561ddeac9e41f54c341b64b4b54d21 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Fri, 18 Oct 2019 17:46:55 +0800 Subject: [PATCH 09/21] fix test_scheduler Former-commit-id: 842753b05bd256a7f4c1433ba7eb3980b595099c --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 434a91846e..a496082f54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-626 - Refactor DataObj to support cache any type data - MS-648 - Improve unittest - MS-655 - Upgrade SPTAG +- \#42 - Put union of index_build_device and search resources to gpu_pool ## New Feature - MS-614 - Preload table at startup From 5cf070237222c1d5749ac94a49f5e3439a01f869 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 18 Oct 2019 19:48:38 +0800 Subject: [PATCH 10/21] format code Former-commit-id: 8c00b569053761c630bbffdb461aae418506207f --- core/src/scheduler/SchedInst.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 1bd1ca00cf..f3f293a0f3 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -76,11 +76,8 @@ load_simple_config() { } if (not find_build_gpu_id) { - ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(build_gpu_id), - "GPU", - build_gpu_id, - true, - true)); + ResMgrInst::GetInstance()->Add( + ResourceFactory::Create(std::to_string(build_gpu_id), "GPU", build_gpu_id, true, true)); ResMgrInst::GetInstance()->Connect("cpu", std::to_string(build_gpu_id), pcie); } } From 11ad9e12fc5e0e2dc47dda5ff5a35c70a195ab01 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 19 Oct 2019 10:22:09 +0800 Subject: [PATCH 11/21] add mysql container to the build environment jenkins slave pod Former-commit-id: ef15e9daf81bbf813ab1170ca52d2473f23c3039 --- ci/jenkinsfile/milvus_build.groovy | 2 +- ci/main_jenkinsfile | 13 +++++++++++++ ci/main_jenkinsfile_no_ut | 13 +++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy index 11f8c51d33..92fd364bb9 100644 --- a/ci/jenkinsfile/milvus_build.groovy +++ b/ci/jenkinsfile/milvus_build.groovy @@ -17,7 +17,7 @@ container('milvus-build-env') { && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.2.1/faiss-branch-0.2.1.tar.gz' \ && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u -c" - sh "./coverage.sh -u root -p 123456 -t 192.168.1.194" + sh "./coverage.sh -u root -p 123456 -t \$POD_IP" } } } catch (exc) { diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 12c6c81cfd..0c3fc32e5b 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -45,6 +45,11 @@ spec: containers: - name: milvus-build-env image: registry.zilliz.com/milvus/milvus-build-env:v0.13 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP command: - cat tty: true @@ -56,6 +61,14 @@ spec: requests: memory: "14Gi" cpu: "5.0" + - name: milvus-mysql + image: mysql:5.6 + env: + - name: MYSQL_ROOT_PASSWORD + value: 123456 + ports: + - containerPort: 3306 + name: mysql """ } } diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index e7382bd1fd..e8d7dae75a 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -45,6 +45,11 @@ spec: containers: - name: milvus-build-env image: registry.zilliz.com/milvus/milvus-build-env:v0.13 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP command: - cat tty: true @@ -56,6 +61,14 @@ spec: requests: memory: "14Gi" cpu: "5.0" + - name: milvus-mysql + image: mysql:5.6 + env: + - name: MYSQL_ROOT_PASSWORD + value: 123456 + ports: + - containerPort: 3306 + name: mysql """ } } From 0c05d16366917c50a2beb4f2b723f05c7e821b86 Mon Sep 17 00:00:00 2001 From: zhenwu Date: Sat, 19 Oct 2019 14:23:12 +0800 Subject: [PATCH 12/21] update coverage.sh Former-commit-id: 9166b5f4118f9f7f4cf4bde3b8027e2d4199a8ed --- core/coverage.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/core/coverage.sh b/core/coverage.sh index 55c70d7e8d..74f9f4219d 100755 --- a/core/coverage.sh +++ b/core/coverage.sh @@ -114,15 +114,15 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "/usr/*" \ "*/boost/*" \ "*/cmake_build/*_ep-prefix/*" \ - "src/index/cmake_build*" \ - "src/index/thirdparty*" \ - "src/grpc*"\ - "src/metrics/MetricBase.h"\ - "src/server/Server.cpp"\ - "src/server/DBWrapper.cpp"\ - "src/server/grpc_impl/GrpcServer.cpp"\ - "src/utils/easylogging++.h"\ - "src/utils/easylogging++.cc"\ + "*/src/index/cmake_build*" \ + "*/src/index/thirdparty*" \ + "*/src/grpc*" \ + "*/src/metrics/MetricBase.h" \ + "*/src/server/Server.cpp" \ + "*/src/server/DBWrapper.cpp" \ + "*/src/server/grpc_impl/GrpcServer.cpp" \ + "*/src/utils/easylogging++.h" \ + "*/src/utils/easylogging++.cc" # gen html report ${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ From b8605caa40e4e9d9f5a379cf15b3ac58f94c589a Mon Sep 17 00:00:00 2001 From: wxyu Date: Sat, 19 Oct 2019 15:03:40 +0800 Subject: [PATCH 13/21] add log Former-commit-id: 12f5c2eb2903b8285c2d4ae3fb58e02fd10c3f83 --- core/src/db/engine/ExecutionEngineImpl.cpp | 3 +++ core/src/wrapper/VecIndex.h | 2 ++ 2 files changed, 5 insertions(+) diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 862c1026d2..0f893f82c7 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -164,6 +164,9 @@ ExecutionEngineImpl::HybridLoad() const { quantizer_conf->mode = 1; quantizer_conf->gpu_id = best_device_id; auto quantizer = index_->LoadQuantizer(quantizer_conf); + if (quantizer == nullptr) { + ENGINE_LOG_ERROR << "quantizer is nullptr"; + } index_->SetQuantizer(quantizer); auto cache_quantizer = std::make_shared(quantizer); cache::GpuCacheMgr::GetInstance(best_device_id)->InsertItem(key, cache_quantizer); diff --git a/core/src/wrapper/VecIndex.h b/core/src/wrapper/VecIndex.h index 05da9ccc03..99b9bc31db 100644 --- a/core/src/wrapper/VecIndex.h +++ b/core/src/wrapper/VecIndex.h @@ -26,6 +26,7 @@ #include "knowhere/common/Config.h" #include "knowhere/index/vector_index/Quantizer.h" #include "utils/Status.h" +#include "utils/Log.h" namespace milvus { namespace engine { @@ -101,6 +102,7 @@ class VecIndex : public cache::DataObj { //////////////// virtual knowhere::QuantizerPtr LoadQuantizer(const Config& conf) { + ENGINE_LOG_ERROR << "LoadQuantizer virtual funciton called."; return nullptr; } From ad6a7348f1b5a2a66501170209e07ecc6de53e58 Mon Sep 17 00:00:00 2001 From: wxyu Date: Sat, 19 Oct 2019 16:44:44 +0800 Subject: [PATCH 14/21] fix format Former-commit-id: 950a3cd61802b42b13d956677ce484b5e74fdd7e --- core/src/scheduler/task/SearchTask.cpp | 5 +++++ core/src/wrapper/VecIndex.h | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp index b7a1e211d2..6c662f8e63 100644 --- a/core/src/scheduler/task/SearchTask.cpp +++ b/core/src/scheduler/task/SearchTask.cpp @@ -204,6 +204,11 @@ XSearchTask::Execute() { output_distance.resize(topk * nq); std::string hdr = "job " + std::to_string(search_job->id()) + " nq " + std::to_string(nq) + " topk " + std::to_string(topk); + auto str = R"(s324gagbds +bbvcb\fds +b +fdb\f +e))"; try { // step 2: search diff --git a/core/src/wrapper/VecIndex.h b/core/src/wrapper/VecIndex.h index 99b9bc31db..36104b2107 100644 --- a/core/src/wrapper/VecIndex.h +++ b/core/src/wrapper/VecIndex.h @@ -25,8 +25,8 @@ #include "knowhere/common/BinarySet.h" #include "knowhere/common/Config.h" #include "knowhere/index/vector_index/Quantizer.h" -#include "utils/Status.h" #include "utils/Log.h" +#include "utils/Status.h" namespace milvus { namespace engine { From 964e1d5cfd9b9bc69735bb55b90985eb615d7d3c Mon Sep 17 00:00:00 2001 From: wxyu Date: Sat, 19 Oct 2019 16:46:35 +0800 Subject: [PATCH 15/21] remove unused code Former-commit-id: 7ebbaeb25d5bd40b0d028a035eafe096d8ca20c9 --- core/src/scheduler/task/SearchTask.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp index 6c662f8e63..b7a1e211d2 100644 --- a/core/src/scheduler/task/SearchTask.cpp +++ b/core/src/scheduler/task/SearchTask.cpp @@ -204,11 +204,6 @@ XSearchTask::Execute() { output_distance.resize(topk * nq); std::string hdr = "job " + std::to_string(search_job->id()) + " nq " + std::to_string(nq) + " topk " + std::to_string(topk); - auto str = R"(s324gagbds -bbvcb\fds -b -fdb\f -e))"; try { // step 2: search From e153262f3fb9719dfdf61b83522a784bd45f46a9 Mon Sep 17 00:00:00 2001 From: starlord Date: Sat, 19 Oct 2019 16:48:56 +0800 Subject: [PATCH 16/21] fix config test bug Former-commit-id: aa548a904c54747c22b34c7d847ca87ed32698f1 --- core/unittest/server/CMakeLists.txt | 8 -- core/unittest/server/appendix/log_config.conf | 27 ------ .../server/appendix/server_config.yaml | 43 --------- core/unittest/server/test_config.cpp | 14 ++- .../server/{util_test.cpp => test_util.cpp} | 5 + core/unittest/server/utils.cpp | 94 +++++++++++++++++++ core/unittest/server/utils.h | 31 ++++++ 7 files changed, 136 insertions(+), 86 deletions(-) delete mode 100644 core/unittest/server/appendix/log_config.conf delete mode 100644 core/unittest/server/appendix/server_config.yaml rename core/unittest/server/{util_test.cpp => test_util.cpp} (99%) create mode 100644 core/unittest/server/utils.cpp create mode 100644 core/unittest/server/utils.h diff --git a/core/unittest/server/CMakeLists.txt b/core/unittest/server/CMakeLists.txt index 4420e2a1a7..180dcfa6d5 100644 --- a/core/unittest/server/CMakeLists.txt +++ b/core/unittest/server/CMakeLists.txt @@ -67,11 +67,3 @@ target_link_libraries(test_server ) install(TARGETS test_server DESTINATION unittest) - -configure_file(appendix/server_config.yaml - "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/server_config.yaml" - COPYONLY) - -configure_file(appendix/log_config.conf - "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/log_config.conf" - COPYONLY) diff --git a/core/unittest/server/appendix/log_config.conf b/core/unittest/server/appendix/log_config.conf deleted file mode 100644 index 0a3e0d21af..0000000000 --- a/core/unittest/server/appendix/log_config.conf +++ /dev/null @@ -1,27 +0,0 @@ -* GLOBAL: - FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log" - ENABLED = true - TO_FILE = true - TO_STANDARD_OUTPUT = false - SUBSECOND_PRECISION = 3 - PERFORMANCE_TRACKING = false - MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB -* DEBUG: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log" - ENABLED = true -* WARNING: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log" -* TRACE: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log" -* VERBOSE: - FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" - TO_FILE = false - TO_STANDARD_OUTPUT = false -## Error logs -* ERROR: - ENABLED = true - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log" -* FATAL: - ENABLED = true - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log" diff --git a/core/unittest/server/appendix/server_config.yaml b/core/unittest/server/appendix/server_config.yaml deleted file mode 100644 index c5aef82511..0000000000 --- a/core/unittest/server/appendix/server_config.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Default values are used when you make no changes to the following parameters. - -server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # port range: 1025 ~ 65534 - deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable - time_zone: UTC+8 - -db_config: - primary_path: /tmp/milvus # path used to store data and meta - secondary_path: # path used to store data only, split by semicolon - - backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database - # Keep 'dialect://:@:/', and replace other texts with real values - # Replace 'dialect' with 'mysql' or 'sqlite' - - insert_buffer_size: 4 # GB, maximum insert buffer size allowed - # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory - - preload_table: # preload data at startup, '*' means load all tables, empty value means no preload - # you can specify preload tables like this: table1,table2,table3 - -metric_config: - enable_monitor: false # enable monitoring or not - collector: prometheus # prometheus - prometheus_config: - port: 8080 # port prometheus uses to fetch metrics - -cache_config: - cpu_cache_capacity: 16 # GB, CPU memory used for cache - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - gpu_cache_capacity: 4 # GB, GPU memory used for cache - gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - cache_insert_data: false # whether to load inserted data into cache - -engine_config: - use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times - # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times - -resource_config: - search_resources: # define the GPUs used for search computation, valid value: gpux - - gpu0 - index_build_device: gpu0 # GPU used for building index diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index a6c6be64c4..d408fdee8c 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -22,28 +22,26 @@ #include "utils/CommonUtil.h" #include "utils/ValidationUtil.h" #include "server/Config.h" +#include "server/utils.h" namespace { -static const char *CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml"; -static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf"; - static constexpr uint64_t KB = 1024; static constexpr uint64_t MB = KB * 1024; static constexpr uint64_t GB = MB * 1024; } // namespace -TEST(ConfigTest, CONFIG_TEST) { +TEST_F(ConfigTest, CONFIG_TEST) { milvus::server::ConfigMgr *config_mgr = milvus::server::YamlConfigMgr::GetInstance(); milvus::Status s = config_mgr->LoadConfigFile(""); ASSERT_FALSE(s.ok()); - s = config_mgr->LoadConfigFile(LOG_FILE_PATH); + s = config_mgr->LoadConfigFile(INVALID_CONFIG_PATH); ASSERT_FALSE(s.ok()); - s = config_mgr->LoadConfigFile(CONFIG_FILE_PATH); + s = config_mgr->LoadConfigFile(VALID_CONFIG_PATH); ASSERT_TRUE(s.ok()); config_mgr->Print(); @@ -99,9 +97,9 @@ TEST(ConfigTest, CONFIG_TEST) { ASSERT_TRUE(seqs.empty()); } -TEST(ConfigTest, SERVER_CONFIG_TEST) { +TEST_F(ConfigTest, SERVER_CONFIG_TEST) { milvus::server::Config &config = milvus::server::Config::GetInstance(); - milvus::Status s = config.LoadConfigFile(CONFIG_FILE_PATH); + milvus::Status s = config.LoadConfigFile(VALID_CONFIG_PATH); ASSERT_TRUE(s.ok()); s = config.ValidateConfig(); diff --git a/core/unittest/server/util_test.cpp b/core/unittest/server/test_util.cpp similarity index 99% rename from core/unittest/server/util_test.cpp rename to core/unittest/server/test_util.cpp index 395839a8c0..24482740bc 100644 --- a/core/unittest/server/util_test.cpp +++ b/core/unittest/server/test_util.cpp @@ -275,6 +275,11 @@ TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) { ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::INVALID).code(), milvus::SERVER_INVALID_INDEX_TYPE); for (int i = 1; i <= (int)milvus::engine::EngineType::MAX_VALUE; i++) { +#ifndef CUSTOMIZATION + if (i == (int)milvus::engine::EngineType::FAISS_IVFSQ8H) { + continue; + } +#endif ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS); } ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType( diff --git a/core/unittest/server/utils.cpp b/core/unittest/server/utils.cpp new file mode 100644 index 0000000000..f79c25bcce --- /dev/null +++ b/core/unittest/server/utils.cpp @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/utils.h" + +#include +#include +#include +#include + +namespace { + +static const char *VALID_CONFIG_STR = "# Default values are used when you make no changes to the following parameters.\n" + "\n" + "server_config:\n" + " address: 0.0.0.0 # milvus server ip address (IPv4)\n" + " port: 19530 # port range: 1025 ~ 65534\n" + " deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable\n" + " time_zone: UTC+8\n" + "\n" + "db_config:\n" + " primary_path: /tmp/milvus # path used to store data and meta\n" + " secondary_path: # path used to store data only, split by semicolon\n" + "\n" + " backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database\n" + " # Keep 'dialect://:@:/', and replace other texts with real values\n" + " # Replace 'dialect' with 'mysql' or 'sqlite'\n" + "\n" + " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" + " # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory\n" + " build_index_gpu: 0 # gpu id used for building index\n" + "\n" + " preload_table: # preload data at startup, '*' means load all tables, empty value means no preload\n" + " # you can specify preload tables like this: table1,table2,table3\n" + "\n" + "metric_config:\n" + " enable_monitor: false # enable monitoring or not\n" + " collector: prometheus # prometheus\n" + " prometheus_config:\n" + " port: 8080 # port prometheus uses to fetch metrics\n" + "\n" + "cache_config:\n" + " cpu_cache_capacity: 16 # GB, CPU memory used for cache\n" + " cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered\n" + " gpu_cache_capacity: 4 # GB, GPU memory used for cache\n" + " gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered\n" + " cache_insert_data: false # whether to load inserted data into cache\n" + "\n" + "engine_config:\n" + " use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times\n" + " # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times\n" + "\n" + "resource_config:\n" + " resource_pool:\n" + " - cpu\n" + " - gpu0"; + +static const char *INVALID_CONFIG_STR = "*INVALID*"; + +void WriteToFile(const char* file_path, const char *content) { + boost::filesystem::path fpath(file_path); + boost::filesystem::fstream fstream(fpath, std::ios_base::out); + + //write data to file + fstream << content; + fstream.close(); +} + +} // namespace + + +void ConfigTest::SetUp() { + WriteToFile(VALID_CONFIG_PATH, VALID_CONFIG_STR); + WriteToFile(INVALID_CONFIG_PATH, INVALID_CONFIG_STR); +} + +void ConfigTest::TearDown() { + boost::filesystem::remove(VALID_CONFIG_PATH); + boost::filesystem::remove(INVALID_CONFIG_PATH); +} \ No newline at end of file diff --git a/core/unittest/server/utils.h b/core/unittest/server/utils.h new file mode 100644 index 0000000000..dc8f0e1a92 --- /dev/null +++ b/core/unittest/server/utils.h @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include +#include + +static const char *VALID_CONFIG_PATH = "/tmp/milvus_test/valid_config.yaml"; +static const char *INVALID_CONFIG_PATH = "/tmp/milvus_test/invalid_config.conf"; + +class ConfigTest : public ::testing::Test { + protected: + void SetUp() override; + void TearDown() override; +}; From e621dfd1e7cd6562f9b05db1a0162b663742108b Mon Sep 17 00:00:00 2001 From: starlord Date: Sat, 19 Oct 2019 17:04:02 +0800 Subject: [PATCH 17/21] #48 Config unittest failed Former-commit-id: 6590479cac96b17485104da98c35d322b563f573 --- CHANGELOG.md | 3 +- core/unittest/server/utils.cpp | 85 +++++++++++++++++----------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58b2d0837f..fa2116a63b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,13 +26,14 @@ Please mark all change in change log and use the ticket from JIRA. - MS-653 - When config check fail, Milvus close without message - MS-654 - Describe index timeout when building index - MS-658 - Fix SQ8 Hybrid can't search -- \#9 Change default gpu_cache_capacity to 4 - MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources +- \#9 - Change default gpu_cache_capacity to 4 - \#20 - C++ sdk example get grpc error - \#23 - Add unittest to improve code coverage - \#31 - make clang-format failed after run build.sh -l - \#39 - Create SQ8H index hang if using github server version - \#30 - Some troubleshoot messages in Milvus do not provide enough information +- \#48 - Config unittest failed ## Improvement - MS-552 - Add and change the easylogging library diff --git a/core/unittest/server/utils.cpp b/core/unittest/server/utils.cpp index f79c25bcce..b9a6433c0b 100644 --- a/core/unittest/server/utils.cpp +++ b/core/unittest/server/utils.cpp @@ -25,49 +25,48 @@ namespace { static const char *VALID_CONFIG_STR = "# Default values are used when you make no changes to the following parameters.\n" - "\n" - "server_config:\n" - " address: 0.0.0.0 # milvus server ip address (IPv4)\n" - " port: 19530 # port range: 1025 ~ 65534\n" - " deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable\n" - " time_zone: UTC+8\n" - "\n" - "db_config:\n" - " primary_path: /tmp/milvus # path used to store data and meta\n" - " secondary_path: # path used to store data only, split by semicolon\n" - "\n" - " backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database\n" - " # Keep 'dialect://:@:/', and replace other texts with real values\n" - " # Replace 'dialect' with 'mysql' or 'sqlite'\n" - "\n" - " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" - " # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory\n" - " build_index_gpu: 0 # gpu id used for building index\n" - "\n" - " preload_table: # preload data at startup, '*' means load all tables, empty value means no preload\n" - " # you can specify preload tables like this: table1,table2,table3\n" - "\n" - "metric_config:\n" - " enable_monitor: false # enable monitoring or not\n" - " collector: prometheus # prometheus\n" - " prometheus_config:\n" - " port: 8080 # port prometheus uses to fetch metrics\n" - "\n" - "cache_config:\n" - " cpu_cache_capacity: 16 # GB, CPU memory used for cache\n" - " cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered\n" - " gpu_cache_capacity: 4 # GB, GPU memory used for cache\n" - " gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered\n" - " cache_insert_data: false # whether to load inserted data into cache\n" - "\n" - "engine_config:\n" - " use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times\n" - " # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times\n" - "\n" - "resource_config:\n" - " resource_pool:\n" - " - cpu\n" - " - gpu0"; + "\n" + "server_config:\n" + " address: 0.0.0.0 # milvus server ip address (IPv4)\n" + " port: 19530 # port range: 1025 ~ 65534\n" + " deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable\n" + " time_zone: UTC+8\n" + "\n" + "db_config:\n" + " primary_path: /tmp/milvus # path used to store data and meta\n" + " secondary_path: # path used to store data only, split by semicolon\n" + "\n" + " backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database\n" + " # Keep 'dialect://:@:/', and replace other texts with real values\n" + " # Replace 'dialect' with 'mysql' or 'sqlite'\n" + "\n" + " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" + " # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory\n" + "\n" + " preload_table: # preload data at startup, '*' means load all tables, empty value means no preload\n" + " # you can specify preload tables like this: table1,table2,table3\n" + "\n" + "metric_config:\n" + " enable_monitor: false # enable monitoring or not\n" + " collector: prometheus # prometheus\n" + " prometheus_config:\n" + " port: 8080 # port prometheus uses to fetch metrics\n" + "\n" + "cache_config:\n" + " cpu_cache_capacity: 16 # GB, CPU memory used for cache\n" + " cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered\n" + " gpu_cache_capacity: 4 # GB, GPU memory used for cache\n" + " gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered\n" + " cache_insert_data: false # whether to load inserted data into cache\n" + "\n" + "engine_config:\n" + " use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times\n" + " # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times\n" + "\n" + "resource_config:\n" + " search_resources: # define the GPUs used for search computation, valid value: gpux\n" + " - gpu0\n" + " index_build_device: gpu0 # GPU used for building index"; static const char *INVALID_CONFIG_STR = "*INVALID*"; From ff27d56571f0eb204a782bc3a092ee760d953a31 Mon Sep 17 00:00:00 2001 From: starlord Date: Sat, 19 Oct 2019 17:08:50 +0800 Subject: [PATCH 18/21] format code Former-commit-id: 633b98f2bae4b9ab323ca2e3a74fc080bfb2f909 --- core/unittest/server/utils.cpp | 94 +++++++++++++++++----------------- 1 file changed, 46 insertions(+), 48 deletions(-) diff --git a/core/unittest/server/utils.cpp b/core/unittest/server/utils.cpp index b9a6433c0b..335ec1300b 100644 --- a/core/unittest/server/utils.cpp +++ b/core/unittest/server/utils.cpp @@ -24,53 +24,49 @@ namespace { -static const char *VALID_CONFIG_STR = "# Default values are used when you make no changes to the following parameters.\n" - "\n" - "server_config:\n" - " address: 0.0.0.0 # milvus server ip address (IPv4)\n" - " port: 19530 # port range: 1025 ~ 65534\n" - " deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable\n" - " time_zone: UTC+8\n" - "\n" - "db_config:\n" - " primary_path: /tmp/milvus # path used to store data and meta\n" - " secondary_path: # path used to store data only, split by semicolon\n" - "\n" - " backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database\n" - " # Keep 'dialect://:@:/', and replace other texts with real values\n" - " # Replace 'dialect' with 'mysql' or 'sqlite'\n" - "\n" - " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" - " # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory\n" - "\n" - " preload_table: # preload data at startup, '*' means load all tables, empty value means no preload\n" - " # you can specify preload tables like this: table1,table2,table3\n" - "\n" - "metric_config:\n" - " enable_monitor: false # enable monitoring or not\n" - " collector: prometheus # prometheus\n" - " prometheus_config:\n" - " port: 8080 # port prometheus uses to fetch metrics\n" - "\n" - "cache_config:\n" - " cpu_cache_capacity: 16 # GB, CPU memory used for cache\n" - " cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered\n" - " gpu_cache_capacity: 4 # GB, GPU memory used for cache\n" - " gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered\n" - " cache_insert_data: false # whether to load inserted data into cache\n" - "\n" - "engine_config:\n" - " use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times\n" - " # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times\n" - "\n" - "resource_config:\n" - " search_resources: # define the GPUs used for search computation, valid value: gpux\n" - " - gpu0\n" - " index_build_device: gpu0 # GPU used for building index"; +static const char + * VALID_CONFIG_STR = "# Default values are used when you make no changes to the following parameters.\n" + "\n" + "server_config:\n" + " address: 0.0.0.0 # milvus server ip address (IPv4)\n" + " port: 19530 # port range: 1025 ~ 65534\n" + " deploy_mode: single \n" + " time_zone: UTC+8\n" + "\n" + "db_config:\n" + " primary_path: /tmp/milvus # path used to store data and meta\n" + " secondary_path: # path used to store data only, split by semicolon\n" + "\n" + " backend_url: sqlite://:@:/ \n" + "\n" + " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" + " preload_table: \n" + "\n" + "metric_config:\n" + " enable_monitor: false # enable monitoring or not\n" + " collector: prometheus # prometheus\n" + " prometheus_config:\n" + " port: 8080 # port prometheus uses to fetch metrics\n" + "\n" + "cache_config:\n" + " cpu_cache_capacity: 16 # GB, CPU memory used for cache\n" + " cpu_cache_threshold: 0.85 \n" + " gpu_cache_capacity: 4 # GB, GPU memory used for cache\n" + " gpu_cache_threshold: 0.85 \n" + " cache_insert_data: false # whether to load inserted data into cache\n" + "\n" + "engine_config:\n" + " use_blas_threshold: 20 \n" + "\n" + "resource_config:\n" + " search_resources: \n" + " - gpu0\n" + " index_build_device: gpu0 # GPU used for building index"; -static const char *INVALID_CONFIG_STR = "*INVALID*"; +static const char* INVALID_CONFIG_STR = "*INVALID*"; -void WriteToFile(const char* file_path, const char *content) { +void +WriteToFile(const char* file_path, const char* content) { boost::filesystem::path fpath(file_path); boost::filesystem::fstream fstream(fpath, std::ios_base::out); @@ -82,12 +78,14 @@ void WriteToFile(const char* file_path, const char *content) { } // namespace -void ConfigTest::SetUp() { +void +ConfigTest::SetUp() { WriteToFile(VALID_CONFIG_PATH, VALID_CONFIG_STR); WriteToFile(INVALID_CONFIG_PATH, INVALID_CONFIG_STR); } -void ConfigTest::TearDown() { +void +ConfigTest::TearDown() { boost::filesystem::remove(VALID_CONFIG_PATH); boost::filesystem::remove(INVALID_CONFIG_PATH); -} \ No newline at end of file +} From 73d2f7c765f0661813191917fd1b815a38a88e1c Mon Sep 17 00:00:00 2001 From: starlord Date: Sat, 19 Oct 2019 17:52:51 +0800 Subject: [PATCH 19/21] fix jenkins build error Former-commit-id: bb994c21d69fb9f8c2c6c57e5389b6133e662c0d --- core/unittest/server/utils.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/unittest/server/utils.cpp b/core/unittest/server/utils.cpp index 335ec1300b..34c98bea16 100644 --- a/core/unittest/server/utils.cpp +++ b/core/unittest/server/utils.cpp @@ -17,6 +17,7 @@ #include "server/utils.h" +#include #include #include #include @@ -67,12 +68,11 @@ static const char* INVALID_CONFIG_STR = "*INVALID*"; void WriteToFile(const char* file_path, const char* content) { - boost::filesystem::path fpath(file_path); - boost::filesystem::fstream fstream(fpath, std::ios_base::out); + std::fstream fs(file_path, std::ios_base::out); //write data to file - fstream << content; - fstream.close(); + fs << content; + fs.close(); } } // namespace From 6e11f4e4e7bdc078ef566863ba2059734a570711 Mon Sep 17 00:00:00 2001 From: starlord Date: Sat, 19 Oct 2019 19:31:22 +0800 Subject: [PATCH 20/21] #48 Config unittest failed Former-commit-id: e07a067382ed00efc9c5d914b4100592bb7aee44 --- core/unittest/server/test_config.cpp | 8 +++++--- core/unittest/server/utils.cpp | 16 +++++++++------- core/unittest/server/utils.h | 5 +++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index d408fdee8c..f3adf8a2c3 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -38,10 +38,11 @@ TEST_F(ConfigTest, CONFIG_TEST) { milvus::Status s = config_mgr->LoadConfigFile(""); ASSERT_FALSE(s.ok()); - s = config_mgr->LoadConfigFile(INVALID_CONFIG_PATH); + std::string config_path(CONFIG_PATH); + s = config_mgr->LoadConfigFile(config_path+ INVALID_CONFIG_FILE); ASSERT_FALSE(s.ok()); - s = config_mgr->LoadConfigFile(VALID_CONFIG_PATH); + s = config_mgr->LoadConfigFile(config_path + VALID_CONFIG_FILE); ASSERT_TRUE(s.ok()); config_mgr->Print(); @@ -98,8 +99,9 @@ TEST_F(ConfigTest, CONFIG_TEST) { } TEST_F(ConfigTest, SERVER_CONFIG_TEST) { + std::string config_path(CONFIG_PATH); milvus::server::Config &config = milvus::server::Config::GetInstance(); - milvus::Status s = config.LoadConfigFile(VALID_CONFIG_PATH); + milvus::Status s = config.LoadConfigFile(config_path + VALID_CONFIG_FILE); ASSERT_TRUE(s.ok()); s = config.ValidateConfig(); diff --git a/core/unittest/server/utils.cpp b/core/unittest/server/utils.cpp index 34c98bea16..4c03da6ad9 100644 --- a/core/unittest/server/utils.cpp +++ b/core/unittest/server/utils.cpp @@ -16,12 +16,12 @@ // under the License. #include "server/utils.h" +#include "utils/CommonUtil.h" #include #include #include #include -#include namespace { @@ -67,8 +67,8 @@ static const char static const char* INVALID_CONFIG_STR = "*INVALID*"; void -WriteToFile(const char* file_path, const char* content) { - std::fstream fs(file_path, std::ios_base::out); +WriteToFile(const std::string& file_path, const char* content) { + std::fstream fs(file_path.c_str(), std::ios_base::out); //write data to file fs << content; @@ -80,12 +80,14 @@ WriteToFile(const char* file_path, const char* content) { void ConfigTest::SetUp() { - WriteToFile(VALID_CONFIG_PATH, VALID_CONFIG_STR); - WriteToFile(INVALID_CONFIG_PATH, INVALID_CONFIG_STR); + std::string config_path(CONFIG_PATH); + milvus::server::CommonUtil::CreateDirectory(config_path); + WriteToFile(config_path + VALID_CONFIG_FILE, VALID_CONFIG_STR); + WriteToFile(config_path+ INVALID_CONFIG_FILE, INVALID_CONFIG_STR); } void ConfigTest::TearDown() { - boost::filesystem::remove(VALID_CONFIG_PATH); - boost::filesystem::remove(INVALID_CONFIG_PATH); + std::string config_path(CONFIG_PATH); + milvus::server::CommonUtil::DeleteDirectory(config_path); } diff --git a/core/unittest/server/utils.h b/core/unittest/server/utils.h index dc8f0e1a92..2efc5e4120 100644 --- a/core/unittest/server/utils.h +++ b/core/unittest/server/utils.h @@ -21,8 +21,9 @@ #include #include -static const char *VALID_CONFIG_PATH = "/tmp/milvus_test/valid_config.yaml"; -static const char *INVALID_CONFIG_PATH = "/tmp/milvus_test/invalid_config.conf"; +static const char *CONFIG_PATH = "/tmp/milvus_test/"; +static const char *VALID_CONFIG_FILE = "valid_config.yaml"; +static const char *INVALID_CONFIG_FILE = "invalid_config.conf"; class ConfigTest : public ::testing::Test { protected: From ac20f4f9a637bcff1cea09128fd4cf56343aa64d Mon Sep 17 00:00:00 2001 From: wxyu Date: Sat, 19 Oct 2019 19:33:06 +0800 Subject: [PATCH 21/21] update Former-commit-id: 99939385bdc36200807099b56fc49d72578489f1 --- core/src/db/engine/ExecutionEngineImpl.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 0f893f82c7..ecd6ff0650 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -124,6 +124,11 @@ ExecutionEngineImpl::HybridLoad() const { return; } + if (index_->GetType() == IndexType::FAISS_IDMAP) { + ENGINE_LOG_WARNING << "HybridLoad with type FAISS_IDMAP, ignore"; + return; + } + const std::string key = location_ + ".quantizer"; std::vector gpus = scheduler::get_gpu_pool(); @@ -178,6 +183,9 @@ ExecutionEngineImpl::HybridUnset() const { if (index_type_ != EngineType::FAISS_IVFSQ8H) { return; } + if (index_->GetType() == IndexType::FAISS_IDMAP) { + return; + } index_->UnsetQuantizer(); }