From bc93735b18ca160394fc02654b868a1bf1094bb5 Mon Sep 17 00:00:00 2001 From: erdustiggen Date: Thu, 14 Nov 2019 18:48:02 +0100 Subject: [PATCH 01/56] Changed from std::cout to LOG for error messages --- core/src/scheduler/task/BuildIndexTask.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index d8602c141e..daab7c9dec 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -146,8 +146,7 @@ XBuildIndexTask::Execute() { status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" - << std::endl; + ENGINE_LOG_ERROR << "Failed to build index, index file is too large or gpu memory is not enough"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); @@ -179,8 +178,8 @@ XBuildIndexTask::Execute() { status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to persist index file: " << table_file.location_ - << ", possible out of disk space" << std::endl; + ENGINE_LOG_ERROR << "Failed to persist index file: " << table_file.location_ + << ", possible out of disk space"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); From d53385caa7b90eab2bb851d597f87bbe2c42cd43 Mon Sep 17 00:00:00 2001 From: erdustiggen Date: Thu, 14 Nov 2019 20:54:25 +0100 Subject: [PATCH 02/56] clang format fix --- core/src/scheduler/task/BuildIndexTask.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index daab7c9dec..f561fa947d 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -179,7 +179,7 @@ XBuildIndexTask::Execute() { ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; ENGINE_LOG_ERROR << "Failed to persist index file: " << table_file.location_ - << ", possible out of disk space"; + << ", possible out of disk space"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); From 1159880036a01b1b2750889592ad2826b31a470a Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Fri, 15 Nov 2019 17:00:53 +0800 Subject: [PATCH 03/56] ignore proto python --- core/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/.gitignore b/core/.gitignore index 74e41dba6b..8db8df41db 100644 --- a/core/.gitignore +++ b/core/.gitignore @@ -9,3 +9,5 @@ output.info output_new.info server.info *.pyc +src/grpc/python_gen.h +src/grpc/python/ From 29586ba725686b1d9f3244ecc3ec2aac80a62db5 Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 15 Nov 2019 19:06:14 +0800 Subject: [PATCH 04/56] #353 Rename config.h.in to version.h.in --- CHANGELOG.md | 1 + core/CMakeLists.txt | 2 +- core/src/db/meta/MetaTypes.h | 2 +- core/src/main.cpp | 2 +- core/src/sdk/grpc/ClientProxy.cpp | 2 +- core/src/server/Server.cpp | 2 +- core/src/server/grpc_impl/GrpcRequestTask.cpp | 2 +- core/src/{config.h.in => version.h.in} | 0 core/unittest/server/test_rpc.cpp | 2 +- 9 files changed, 8 insertions(+), 7 deletions(-) rename core/src/{config.h.in => version.h.in} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2aab00f8db..76c8a13c45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#248 - Reside src/external in thirdparty - \#316 - Some files not merged after vectors added - \#340 - Test cases run failed on 0.6.0 +- \#353 - Rename config.h.in to version.h.in ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index f24935f788..671330c192 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -70,7 +70,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL "" endif() message(STATUS "Build version = ${MILVUS_VERSION}") -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/config.h @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/version.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/version.h @ONLY) message(STATUS "Milvus version: " "${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} " diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index 28f35e76fc..d98b74be7d 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -19,7 +19,7 @@ #include "db/Constants.h" #include "db/engine/ExecutionEngine.h" -#include "src/config.h" +#include "src/version.h" #include #include diff --git a/core/src/main.cpp b/core/src/main.cpp index 401736c34f..b39ba87997 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -25,7 +25,7 @@ #include "easyloggingpp/easylogging++.h" #include "metrics/Metrics.h" #include "server/Server.h" -#include "src/config.h" +#include "src/version.h" #include "utils/CommonUtil.h" #include "utils/SignalUtil.h" diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 4ec94cfa98..4a84323d64 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -17,7 +17,7 @@ #include "sdk/grpc/ClientProxy.h" #include "grpc/gen-milvus/milvus.grpc.pb.h" -#include "src/config.h" +#include "src/version.h" #include #include diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index eb2b077829..5676504722 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -25,7 +25,7 @@ #include "server/DBWrapper.h" #include "server/Server.h" #include "server/grpc_impl/GrpcServer.h" -#include "src/config.h" +#include "src/version.h" #include "utils/Log.h" #include "utils/LogUtil.h" #include "utils/SignalUtil.h" diff --git a/core/src/server/grpc_impl/GrpcRequestTask.cpp b/core/src/server/grpc_impl/GrpcRequestTask.cpp index b8a9dbccbf..0d685bed4b 100644 --- a/core/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/core/src/server/grpc_impl/GrpcRequestTask.cpp @@ -29,7 +29,7 @@ #include "scheduler/SchedInst.h" #include "server/DBWrapper.h" #include "server/Server.h" -#include "src/config.h" +#include "src/version.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" diff --git a/core/src/config.h.in b/core/src/version.h.in similarity index 100% rename from core/src/config.h.in rename to core/src/version.h.in diff --git a/core/unittest/server/test_rpc.cpp b/core/unittest/server/test_rpc.cpp index 76c10769a7..e491d5e7f3 100644 --- a/core/unittest/server/test_rpc.cpp +++ b/core/unittest/server/test_rpc.cpp @@ -23,7 +23,7 @@ #include "server/grpc_impl/GrpcRequestHandler.h" #include "server/grpc_impl/GrpcRequestScheduler.h" #include "server/grpc_impl/GrpcRequestTask.h" -#include "src/config.h" +#include "src/version.h" #include "grpc/gen-milvus/milvus.grpc.pb.h" #include "grpc/gen-status/status.pb.h" From a7cb907e44e44481e7391a0f08c2654359430e77 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Fri, 15 Nov 2019 20:15:34 +0800 Subject: [PATCH 05/56] remove gpu in unittest --- core/unittest/db/utils.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index f87371fe6b..bcf1462ecd 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -155,12 +155,10 @@ DBTest::SetUp() { res_mgr->Clear(); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - res_mgr->Add(milvus::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); auto default_conn = milvus::scheduler::Connection("IO", 500.0); auto PCIE = milvus::scheduler::Connection("IO", 11000.0); res_mgr->Connect("disk", "cpu", default_conn); - res_mgr->Connect("cpu", "gtx1660", PCIE); res_mgr->Start(); milvus::scheduler::SchedInst::GetInstance()->Start(); From 15315158e790d2be24ba45696bd27b27705aff7f Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Sat, 16 Nov 2019 09:34:17 +0800 Subject: [PATCH 06/56] modify sdk --- .../sdk/examples/partition/src/ClientTest.cpp | 65 ++++++++++--------- core/src/sdk/examples/simple/main.cpp | 2 +- .../sdk/examples/simple/src/ClientTest.cpp | 3 +- core/src/sdk/grpc/ClientProxy.cpp | 30 ++++++--- core/src/sdk/grpc/ClientProxy.h | 2 +- 5 files changed, 58 insertions(+), 44 deletions(-) diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index 775e1f6d60..b67b1fe907 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -31,45 +31,45 @@ namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); + const char *TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; -constexpr int64_t BATCH_ROW_COUNT = 10000; -constexpr int64_t NQ = 5; -constexpr int64_t TOP_K = 10; -constexpr int64_t NPROBE = 32; -constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different -constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; -constexpr int32_t N_LIST = 15000; -constexpr int32_t PARTITION_COUNT = 5; -constexpr int32_t TARGET_PARTITION = 3; + constexpr int64_t TABLE_DIMENSION = 512; + constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; + constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; + constexpr int64_t BATCH_ROW_COUNT = 10000; + constexpr int64_t NQ = 5; + constexpr int64_t TOP_K = 10; + constexpr int64_t NPROBE = 32; + constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different + constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; + constexpr int32_t N_LIST = 15000; + constexpr int32_t PARTITION_COUNT = 5; + constexpr int32_t TARGET_PARTITION = 3; -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; -} + milvus::TableSchema + BuildTableSchema() { + milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; + return tb_schema; + } -milvus::PartitionParam -BuildPartitionParam(int32_t index) { - std::string tag = std::to_string(index); - std::string partition_name = std::string(TABLE_NAME) + "_" + tag; - milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; - return partition_param; -} + milvus::PartitionParam + BuildPartitionParam(int32_t index) { + std::string tag = std::to_string(index); + std::string partition_name = std::string(TABLE_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; + return partition_param; + } -milvus::IndexParam -BuildIndexParam() { - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; - return index_param; -} + milvus::IndexParam + BuildIndexParam() { + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + return index_param; + } } // namespace void -ClientTest::Test(const std::string& address, const std::string& port) { +ClientTest::Test(const std::string &address, const std::string &port) { std::shared_ptr conn = milvus::Connection::Create(); milvus::Status stat; @@ -78,7 +78,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - +#ifdef yhz { // create table milvus::TableSchema tb_schema = BuildTableSchema(); stat = conn->CreateTable(tb_schema); @@ -202,4 +202,5 @@ ClientTest::Test(const std::string& address, const std::string& port) { } milvus::Connection::Destroy(conn); +#endif } diff --git a/core/src/sdk/examples/simple/main.cpp b/core/src/sdk/examples/simple/main.cpp index c08741606c..d9b6194329 100644 --- a/core/src/sdk/examples/simple/main.cpp +++ b/core/src/sdk/examples/simple/main.cpp @@ -36,7 +36,7 @@ main(int argc, char* argv[]) { {nullptr, 0, nullptr, 0}}; int option_index = 0; - std::string address = "127.0.0.1", port = "19530"; + std::string address = "192.168.1.89", port = "19530"; app_name = argv[0]; int value; diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp index dfa5e2219e..da43f2f2aa 100644 --- a/core/src/sdk/examples/simple/src/ClientTest.cpp +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -68,7 +68,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - +#ifdef yhz { // server version std::string version = conn->ServerVersion(); std::cout << "Server version: " << version << std::endl; @@ -206,4 +206,5 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; } +#endif } diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 4ec94cfa98..5e22904a08 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -43,16 +43,28 @@ Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; - channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - if (channel_ != nullptr) { - connected_ = true; - client_ptr_ = std::make_shared(channel_); - return Status::OK(); - } +// channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - std::string reason = "connect failed!"; - connected_ = false; - return Status(StatusCode::NotConnected, reason); +// channel_ = std::make_shared(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); +// channel_ = std::shared_ptr(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); + auto uri_str = uri.c_str(); + grpc_channel * channel = grpc_insecure_channel_create(uri_str, nullptr, nullptr); +// grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr); + auto state = grpc_channel_check_connectivity_state(channel, true); + if (state == GRPC_CHANNEL_READY) { + std::cout << "Connect " << uri << " successfully"; + } else { + std::cout << "Connect " << uri << " failed."; + } +// if (channel_ != nullptr) { +// connected_ = true; +// client_ptr_ = std::make_shared(channel_); +// return Status::OK(); +// } + +// std::string reason = "connect failed!"; +// connected_ = false; +// return Status(StatusCode::NotConnected, reason); } Status diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index e332266acf..572b782769 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -105,7 +105,7 @@ class ClientProxy : public Connection { DropPartition(const PartitionParam& partition_param) override; private: - std::shared_ptr<::grpc::Channel> channel_; + std::shared_ptr channel_; private: std::shared_ptr client_ptr_; From cd80cdba02f489f68dc56980baa07e517ec50916 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Sat, 16 Nov 2019 15:53:45 +0800 Subject: [PATCH 07/56] sdk_simple return empty result --- CHANGELOG.md | 1 + core/src/grpc/gen-milvus/milvus.pb.h | 57 +++++++++++++------ core/src/sdk/grpc/ClientProxy.cpp | 16 +----- core/src/server/grpc_impl/GrpcRequestTask.cpp | 7 +-- 4 files changed, 47 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 222ac5a8be..56adb23e63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#316 - Some files not merged after vectors added - \#327 - Search does not use GPU when index type is FLAT - \#340 - Test cases run failed on 0.6.0 +- \#374 - sdk_simple return empty result ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/grpc/gen-milvus/milvus.pb.h b/core/src/grpc/gen-milvus/milvus.pb.h index 53ed2db22e..536365b23b 100644 --- a/core/src/grpc/gen-milvus/milvus.pb.h +++ b/core/src/grpc/gen-milvus/milvus.pb.h @@ -1321,7 +1321,8 @@ class RowRecord : void clear_vector_data(); float vector_data(int index) const; void set_vector_data(int index, float value); - void add_vector_data(float value); +// void add_vector_data(float value); + void add_vector_data(std::vector::const_iterator begin, std::vector::const_iterator end); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& vector_data() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* @@ -1473,7 +1474,9 @@ class InsertParam : void clear_row_id_array(); ::PROTOBUF_NAMESPACE_ID::int64 row_id_array(int index) const; void set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::int64 value); - void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value); +// void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value); + void add_row_id_array(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, + std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& row_id_array() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* @@ -2136,7 +2139,9 @@ class TopKQueryResult : void clear_ids(); ::PROTOBUF_NAMESPACE_ID::int64 ids(int index) const; void set_ids(int index, ::PROTOBUF_NAMESPACE_ID::int64 value); - void add_ids(::PROTOBUF_NAMESPACE_ID::int64 value); +// void add_ids(::PROTOBUF_NAMESPACE_ID::int64 value); + void add_ids(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, + std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& ids() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* @@ -2147,7 +2152,8 @@ class TopKQueryResult : void clear_distances(); float distances(int index) const; void set_distances(int index, float value); - void add_distances(float value); +// void add_distances(float value); + void add_distances(std::vector::const_iterator begin, std::vector::const_iterator end); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& distances() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* @@ -3922,9 +3928,14 @@ inline void RowRecord::set_vector_data(int index, float value) { vector_data_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.RowRecord.vector_data) } -inline void RowRecord::add_vector_data(float value) { - vector_data_.Add(value); - // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) +//inline void RowRecord::add_vector_data(float value) { +// vector_data_.Add(value); +// // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) +//} +inline void RowRecord::add_vector_data(std::vector::const_iterator begin, + std::vector::const_iterator end) { + vector_data_.Add(begin, end); + // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& RowRecord::vector_data() const { @@ -4037,9 +4048,14 @@ inline void InsertParam::set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::in row_id_array_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.InsertParam.row_id_array) } -inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) { - row_id_array_.Add(value); - // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) +//inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) { +// row_id_array_.Add(value); +// // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) +//} +inline void InsertParam::add_row_id_array(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, + std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end) { + row_id_array_.Add(begin, end); + // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& InsertParam::row_id_array() const { @@ -4588,9 +4604,14 @@ inline void TopKQueryResult::set_ids(int index, ::PROTOBUF_NAMESPACE_ID::int64 v ids_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.TopKQueryResult.ids) } -inline void TopKQueryResult::add_ids(::PROTOBUF_NAMESPACE_ID::int64 value) { - ids_.Add(value); - // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) +//inline void TopKQueryResult::add_ids(::PROTOBUF_NAMESPACE_ID::int64 value) { +// ids_.Add(value); +// // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) +//} +inline void TopKQueryResult::add_ids(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, + std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end) { + ids_.Add(begin,end); + // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& TopKQueryResult::ids() const { @@ -4618,9 +4639,13 @@ inline void TopKQueryResult::set_distances(int index, float value) { distances_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.TopKQueryResult.distances) } -inline void TopKQueryResult::add_distances(float value) { - distances_.Add(value); - // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) +//inline void TopKQueryResult::add_distances(float value) { +// distances_.Add(value); +// // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) +//} +inline void TopKQueryResult::add_distances(std::vector::const_iterator begin, std::vector::const_iterator end) { + distances_.Add(begin, end); + // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& TopKQueryResult::distances() const { diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 4ec94cfa98..eaf29e9c69 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -32,13 +32,6 @@ UriCheck(const std::string& uri) { return (index != std::string::npos); } -void -CopyRowRecord(::milvus::grpc::RowRecord* target, const RowRecord& src) { - auto vector_data = target->mutable_vector_data(); - vector_data->Resize(static_cast(src.data.size()), 0.0); - memcpy(vector_data->mutable_data(), src.data.data(), src.data.size() * sizeof(float)); -} - Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; @@ -196,17 +189,14 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_ for (auto& record : record_array) { ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); - CopyRowRecord(grpc_record, record); + grpc_record->add_vector_data(record.data.begin(), record.data.end()); } // Single thread ::milvus::grpc::VectorIds vector_ids; if (!id_array.empty()) { /* set user's ids */ - auto row_ids = insert_param.mutable_row_id_array(); - row_ids->Reserve(static_cast(id_array.size())); - memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); - + insert_param.add_row_id_array(id_array.begin(), id_array.end()); client_ptr_->Insert(vector_ids, insert_param, status); } else { client_ptr_->Insert(vector_ids, insert_param, status); @@ -236,7 +226,7 @@ ClientProxy::Search(const std::string& table_name, const std::vectoradd_vector_data(record.data.begin(), record.data.end()); } // step 2: convert range array diff --git a/core/src/server/grpc_impl/GrpcRequestTask.cpp b/core/src/server/grpc_impl/GrpcRequestTask.cpp index b8a9dbccbf..c54e9dd388 100644 --- a/core/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/core/src/server/grpc_impl/GrpcRequestTask.cpp @@ -682,11 +682,8 @@ SearchTask::OnExecute() { // step 7: construct result array topk_result_->set_row_num(record_count); - topk_result_->mutable_ids()->Resize(static_cast(result_ids.size()), 0); - memcpy(topk_result_->mutable_ids()->mutable_data(), result_ids.data(), result_ids.size() * sizeof(int64_t)); - topk_result_->mutable_distances()->Resize(static_cast(result_distances.size()), 0.0); - memcpy(topk_result_->mutable_distances()->mutable_data(), result_distances.data(), - result_distances.size() * sizeof(float)); + topk_result_->add_ids(result_ids.begin(), result_ids.end()); + topk_result_->add_distances(result_distances.begin(), result_distances.end()); // step 8: print time cost percent rc.RecordSection("construct result and send"); From 394b763278e87fa8014211943ba3b11bbc996b13 Mon Sep 17 00:00:00 2001 From: zhenwu Date: Sat, 16 Nov 2019 18:13:31 +0800 Subject: [PATCH 08/56] Update case --- tests/milvus_python_test/conftest.py | 16 +++++++++++----- tests/milvus_python_test/test_connect.py | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/milvus_python_test/conftest.py b/tests/milvus_python_test/conftest.py index 8bab824606..dc58063a45 100644 --- a/tests/milvus_python_test/conftest.py +++ b/tests/milvus_python_test/conftest.py @@ -42,16 +42,22 @@ def connect(request): port = request.config.getoption("--port") milvus = Milvus() try: - milvus.connect(host=ip, port=port) - except: + status = milvus.connect(host=ip, port=port) + logging.getLogger().info(status) + if not status.OK(): + # try again + logging.getLogger().info("------------------------------------") + logging.getLogger().info("Try to connect again") + logging.getLogger().info("------------------------------------") + res = milvus.connect(host=ip, port=port) + except Exception as e: + logging.getLogger().error(str(e)) pytest.exit("Milvus server can not connected, exit pytest ...") - def fin(): try: milvus.disconnect() except: pass - request.addfinalizer(fin) return milvus @@ -129,4 +135,4 @@ def ip_table(request, connect): request.addfinalizer(teardown) - return table_name \ No newline at end of file + return table_name diff --git a/tests/milvus_python_test/test_connect.py b/tests/milvus_python_test/test_connect.py index 96ce1d3bdf..dd7e80c1f9 100644 --- a/tests/milvus_python_test/test_connect.py +++ b/tests/milvus_python_test/test_connect.py @@ -249,8 +249,8 @@ class TestConnect: ''' milvus = Milvus() uri_value = "tcp://%s:19540" % args["ip"] - milvus.connect(host=args["ip"], port="", uri=uri_value) - assert milvus.connected() + with pytest.raises(Exception) as e: + milvus.connect(host=args["ip"], port="", uri=uri_value) def test_connect_param_priority_uri(self, args): ''' From 4c71748bb2b423fc56a9c63c44c2f3acefae371b Mon Sep 17 00:00:00 2001 From: erdustiggen Date: Sat, 16 Nov 2019 20:55:16 +0100 Subject: [PATCH 09/56] Fix typos and grammatical errors --- core/src/wrapper/VecImpl.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index c97900f839..931eef9dbf 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -26,7 +26,7 @@ /* * no parameter check in this layer. - * only responible for index combination + * only responsible for index combination */ namespace milvus { @@ -271,7 +271,7 @@ IVFHybridIndex::LoadQuantizer(const Config& conf) { if (auto new_idx = std::dynamic_pointer_cast(index_)) { return new_idx->LoadQuantizer(conf); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); } } @@ -282,8 +282,8 @@ IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) { if (auto new_idx = std::dynamic_pointer_cast(index_)) { new_idx->SetQuantizer(q); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); - return Status(KNOWHERE_ERROR, "not support"); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + return Status(KNOWHERE_ERROR, "not supported"); } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); @@ -302,8 +302,8 @@ IVFHybridIndex::UnsetQuantizer() { if (auto new_idx = std::dynamic_pointer_cast(index_)) { new_idx->UnsetQuantizer(); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); - return Status(KNOWHERE_ERROR, "not support"); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + return Status(KNOWHERE_ERROR, "not supported"); } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); @@ -322,7 +322,7 @@ IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { if (auto new_idx = std::dynamic_pointer_cast(index_)) { return std::make_shared(new_idx->LoadData(q, conf), type); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); @@ -341,7 +341,7 @@ IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& c auto new_idx = std::make_shared(pair.first, type); return std::make_pair(new_idx, pair.second); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); From d5eb5675b77210752140184df8a5819edfa92e95 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Mon, 18 Nov 2019 09:53:38 +0800 Subject: [PATCH 10/56] fix by reduce knng size --- core/src/index/CMakeLists.txt | 6 +- .../index/knowhere/knowhere/common/Config.h | 14 ++ .../vector_index/helpers/IndexParameter.cpp | 22 ++++ .../vector_index/helpers/IndexParameter.h | 9 ++ core/src/wrapper/ConfAdapter.cpp | 27 ++-- core/src/wrapper/ConfAdapter.h | 3 - core/unittest/wrapper/test_wrapper.cpp | 121 +++++++++++++++++- core/unittest/wrapper/utils.cpp | 5 + core/unittest/wrapper/utils.h | 10 +- 9 files changed, 188 insertions(+), 29 deletions(-) diff --git a/core/src/index/CMakeLists.txt b/core/src/index/CMakeLists.txt index 4b5c1b1de3..53453d53aa 100644 --- a/core/src/index/CMakeLists.txt +++ b/core/src/index/CMakeLists.txt @@ -88,14 +88,14 @@ endif () include(ThirdPartyPackagesCore) if (CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") if (KNOWHERE_GPU_VERSION) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") endif () else () - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") if (KNOWHERE_GPU_VERSION) - set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3 -g") endif () endif () diff --git a/core/src/index/knowhere/knowhere/common/Config.h b/core/src/index/knowhere/knowhere/common/Config.h index 6191ecb771..48c2de8b1b 100644 --- a/core/src/index/knowhere/knowhere/common/Config.h +++ b/core/src/index/knowhere/knowhere/common/Config.h @@ -18,6 +18,8 @@ #pragma once #include +#include +#include "Log.h" namespace knowhere { @@ -50,6 +52,18 @@ struct Cfg { CheckValid() { return true; } + + void + Dump() { + KNOWHERE_LOG_DEBUG << DumpImpl().str(); + } + + virtual std::stringstream + DumpImpl() { + std::stringstream ss; + ss << "dim: " << d << ", metric: " << int(metric_type) << ", gpuid: " << gpu_id << ", k: " << k; + return ss; + } }; using Config = std::shared_ptr; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp index 20f3388174..6e9ee55658 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp @@ -34,4 +34,26 @@ GetMetricType(METRICTYPE& type) { KNOWHERE_THROW_MSG("Metric type is invalid"); } +std::stringstream +IVFCfg::DumpImpl() { + auto ss = Cfg::DumpImpl(); + ss << ", nlist: " << nlist << ", nprobe: " << nprobe; + return ss; +} + +std::stringstream +IVFSQCfg::DumpImpl() { + auto ss = IVFCfg::DumpImpl(); + ss << ", nbits: " << nbits; + return ss; +} + +std::stringstream +NSGCfg::DumpImpl() { + auto ss = IVFCfg::DumpImpl(); + ss << ", knng: " << knng << ", search_length: " << search_length << ", out_degree: " << out_degree + << ", candidate: " << candidate_pool_size; + return ss; +} + } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h index b2854abef8..b931790b04 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -53,6 +53,9 @@ struct IVFCfg : public Cfg { IVFCfg() = default; + std::stringstream + DumpImpl() override; + bool CheckValid() override { return true; @@ -69,6 +72,9 @@ struct IVFSQCfg : public IVFCfg { : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), nbits(nbits) { } + std::stringstream + DumpImpl() override; + IVFSQCfg() = default; bool @@ -119,6 +125,9 @@ struct NSGCfg : public IVFCfg { NSGCfg() = default; + std::stringstream + DumpImpl() override; + bool CheckValid() override { return true; diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 461745f1fd..b121372a26 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -47,7 +47,8 @@ ConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; + conf->k = metaconf.k; MatchBase(conf); return conf; } @@ -65,7 +66,7 @@ IVFConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; MatchBase(conf); return conf; } @@ -114,7 +115,7 @@ IVFSQConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; MatchBase(conf); return conf; @@ -126,7 +127,7 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; if (!(conf->d % 4)) @@ -175,21 +176,17 @@ NSGConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; + conf->k = metaconf.k; - double factor = metaconf.size / TYPICAL_COUNT; auto scale_factor = round(metaconf.dim / 128.0); scale_factor = scale_factor >= 4 ? 4 : scale_factor; - conf->nprobe = conf->nlist > 10000 ? conf->nlist * 0.02 : conf->nlist * 0.1; - conf->knng = (100 + 100 * scale_factor) * factor; - conf->search_length = (40 + 5 * scale_factor) * factor; - conf->out_degree = (50 + 5 * scale_factor) * factor; - conf->candidate_pool_size = (200 + 100 * scale_factor) * factor; + conf->nprobe = int64_t(conf->nlist * 0.01); + conf->knng = 40 + 10 * scale_factor; // the size of knng + conf->search_length = 40 + 5 * scale_factor; + conf->out_degree = 50 + 5 * scale_factor; + conf->candidate_pool_size = 200 + 100 * scale_factor; MatchBase(conf); - - // WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist - // << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d - // << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng; return conf; } diff --git a/core/src/wrapper/ConfAdapter.h b/core/src/wrapper/ConfAdapter.h index 5ec3d52486..85637a4969 100644 --- a/core/src/wrapper/ConfAdapter.h +++ b/core/src/wrapper/ConfAdapter.h @@ -46,9 +46,6 @@ class ConfAdapter { virtual knowhere::Config MatchSearch(const TempMetaConf& metaconf, const IndexType& type); - // virtual void - // Dump(){} - protected: static void MatchBase(knowhere::Config conf); diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 025601a1cd..11a8b8e01a 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -56,10 +56,6 @@ class KnowhereWrapperTest index_ = GetVecIndexFactory(index_type); conf = ParamGenerator::GetInstance().GenBuild(index_type, tempconf); searchconf = ParamGenerator::GetInstance().GenSearchConf(index_type, tempconf); - -// conf->k = k; -// conf->d = dim; -// conf->gpu_id = DEVICEID; } void TearDown() override { @@ -97,6 +93,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest, TEST_P(KnowhereWrapperTest, BASE_TEST) { EXPECT_EQ(index_->GetType(), index_type); + // conf->Dump(); auto elems = nq * k; std::vector res_ids(elems); @@ -191,3 +188,119 @@ TEST(whatever, test_config) { auto pq_conf = std::make_shared(); pq_conf->Match(conf); } + +// #include "knowhere/index/vector_index/IndexIDMAP.h" +// #include "src/wrapper/VecImpl.h" +// #include "src/index/unittest/utils.h" +// The two case below prove NSG is concern with data distribution +// Further work: 1. Use right basedata and pass it by milvus +// a. batch size is 100000 [Pass] +// b. transfer all at once [Pass] +// 2. Use SIFT1M in test and check time cost [] +// TEST_P(KnowhereWrapperTest, nsgwithidmap) { +// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP); +// auto ori_xb = xb; +// auto ori_ids = ids; +// std::vector temp_xb; +// std::vector temp_ids; +// nb = 50000; +// for (int i = 0; i < 20; ++i) { +// GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis); +// assert(xb.size() == nb*dim); +// //#define IDMAP +// #ifdef IDMAP +// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim); +// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1)); +// if (i == 0) { +// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf); +// } else { +// idmap->Add(nb, temp_xb.data(), temp_ids.data()); +// } +// temp_xb.clear(); +// temp_ids.clear(); +// #else +// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim); +// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1)); +// #endif +// } + +// #ifdef IDMAP +// auto idmap_idx = std::dynamic_pointer_cast(idmap); +// auto x = idmap_idx->Count(); +// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf); +// #else +// assert(temp_xb.size() == 1000000*128); +// index_->BuildAll(1000000, temp_xb.data(), ori_ids.data(), conf); +// #endif +// } + +// TEST_P(KnowhereWrapperTest, nsgwithsidmap) { +// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP); +// auto ori_xb = xb; +// std::vector temp_xb; +// std::vector temp_ids; +// nb = 50000; +// for (int i = 0; i < 20; ++i) { +// #define IDMAP +// #ifdef IDMAP +// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1)); +// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1)); +// if (i == 0) { +// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf); +// } else { +// idmap->Add(nb, temp_xb.data(), temp_ids.data()); +// } +// temp_xb.clear(); +// temp_ids.clear(); +// #else +// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1)); +// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1)); +// #endif +// } + +// #ifdef IDMAP +// auto idmap_idx = std::dynamic_pointer_cast(idmap); +// auto x = idmap_idx->Count(); +// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf); +// #else +// index_->BuildAll(1000000, temp_xb.data(), temp_ids.data(), conf); +// #endif + +// // The code use to store raw base data +// FileIOWriter writer("/tmp/newraw"); +// ori_xb.shrink_to_fit(); +// std::cout << "size" << ori_xb.size(); +// writer(static_cast(ori_xb.data()), ori_xb.size()* sizeof(float)); +// std::cout << "Finish!" << std::endl; +// } + +// void load_data(char* filename, float*& data, unsigned& num, +// unsigned& dim) { // load data with sift10K pattern +// std::ifstream in(filename, std::ios::binary); +// if (!in.is_open()) { +// std::cout << "open file error" << std::endl; +// exit(-1); +// } +// in.read((char*)&dim, 4); +// in.seekg(0, std::ios::end); +// std::ios::pos_type ss = in.tellg(); +// size_t fsize = (size_t)ss; +// num = (unsigned)(fsize / (dim + 1) / 4); +// data = new float[(size_t)num * (size_t)dim]; + +// in.seekg(0, std::ios::beg); +// for (size_t i = 0; i < num; i++) { +// in.seekg(4, std::ios::cur); +// in.read((char*)(data + i * dim), dim * 4); +// } +// in.close(); +// } + +// TEST_P(KnowhereWrapperTest, Sift1M) { +// float* data = nullptr; +// unsigned points_num, dim; +// load_data("/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Data/sift/sift_base.fvecs", data, points_num, +// dim); std::cout << points_num << " " << dim << std::endl; + +// index_->BuildAll(points_num, data, ids.data(), conf); +// } \ No newline at end of file diff --git a/core/unittest/wrapper/utils.cpp b/core/unittest/wrapper/utils.cpp index 00a1db2c88..96b9e643f5 100644 --- a/core/unittest/wrapper/utils.cpp +++ b/core/unittest/wrapper/utils.cpp @@ -117,6 +117,11 @@ void DataGenBase::GenData(const int& dim, const int& nb, const int& nq, std::vector& xb, std::vector& xq, std::vector& ids, const int& k, std::vector& gt_ids, std::vector& gt_dis) { + xb.clear(); + xq.clear(); + ids.clear(); + gt_ids.clear(); + gt_dis.clear(); xb.resize(nb * dim); xq.resize(nq * dim); ids.resize(nb); diff --git a/core/unittest/wrapper/utils.h b/core/unittest/wrapper/utils.h index 1eaa7ae4ec..cc1926302d 100644 --- a/core/unittest/wrapper/utils.h +++ b/core/unittest/wrapper/utils.h @@ -25,7 +25,6 @@ #include #include - #include "wrapper/VecIndex.h" #include "wrapper/utils.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" @@ -90,17 +89,20 @@ class ParamGenerator { return instance; } - knowhere::Config GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { + knowhere::Config + GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->MatchSearch(conf, type); } - knowhere::Config GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { + knowhere::Config + GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->Match(conf); } - knowhere::Config Gen(const milvus::engine::IndexType& type) { + knowhere::Config + Gen(const milvus::engine::IndexType& type) { switch (type) { case milvus::engine::IndexType::FAISS_IDMAP: { auto tempconf = std::make_shared(); From 82cb637ca8760828e3095783ca0b600d165c90d1 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Mon, 18 Nov 2019 10:50:20 +0800 Subject: [PATCH 11/56] code style format --- core/src/wrapper/ConfAdapter.cpp | 2 +- core/unittest/wrapper/test_wrapper.cpp | 2 +- core/unittest/wrapper/utils.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index b121372a26..0ab38d3394 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -182,7 +182,7 @@ NSGConfAdapter::Match(const TempMetaConf& metaconf) { auto scale_factor = round(metaconf.dim / 128.0); scale_factor = scale_factor >= 4 ? 4 : scale_factor; conf->nprobe = int64_t(conf->nlist * 0.01); - conf->knng = 40 + 10 * scale_factor; // the size of knng + conf->knng = 40 + 10 * scale_factor; // the size of knng conf->search_length = 40 + 5 * scale_factor; conf->out_degree = 50 + 5 * scale_factor; conf->candidate_pool_size = 200 + 100 * scale_factor; diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 11a8b8e01a..a07fafc7b8 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -303,4 +303,4 @@ TEST(whatever, test_config) { // dim); std::cout << points_num << " " << dim << std::endl; // index_->BuildAll(points_num, data, ids.data(), conf); -// } \ No newline at end of file +// } diff --git a/core/unittest/wrapper/utils.h b/core/unittest/wrapper/utils.h index cc1926302d..05dc92f2d6 100644 --- a/core/unittest/wrapper/utils.h +++ b/core/unittest/wrapper/utils.h @@ -89,13 +89,13 @@ class ParamGenerator { return instance; } - knowhere::Config + knowhere::Config GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->MatchSearch(conf, type); } - knowhere::Config + knowhere::Config GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->Match(conf); From 513ad3b8424d27f886fd6162e8fea6f611be5293 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Mon, 18 Nov 2019 11:38:48 +0800 Subject: [PATCH 12/56] Support build index with multiple gpu --- CHANGELOG.md | 1 + core/conf/server_cpu_config.template | 3 +- core/conf/server_gpu_config.template | 3 +- core/src/db/engine/ExecutionEngineImpl.cpp | 11 +- core/src/scheduler/JobMgr.cpp | 2 +- core/src/scheduler/SchedInst.cpp | 33 ++++-- core/src/scheduler/SchedInst.h | 5 + .../scheduler/action/PushTaskToNeighbour.cpp | 102 ++++++------------ .../scheduler/optimizer/BuildIndexPass.cpp | 46 ++++++++ core/src/scheduler/optimizer/BuildIndexPass.h | 51 +++++++++ core/src/scheduler/optimizer/OnlyGPUPass.cpp | 3 +- core/src/server/Config.cpp | 72 ++++++++----- core/src/server/Config.h | 7 +- core/src/wrapper/KnowhereResource.cpp | 8 +- core/unittest/server/test_config.cpp | 3 +- 15 files changed, 233 insertions(+), 117 deletions(-) create mode 100644 core/src/scheduler/optimizer/BuildIndexPass.cpp create mode 100644 core/src/scheduler/optimizer/BuildIndexPass.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 745310faa7..42d9291573 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#12 - Pure CPU version for Milvus - \#77 - Support table partition - \#226 - Experimental shards middleware for Milvus +- \#346 - Support build index with multiple gpu ## Improvement - \#275 - Rename C++ SDK IndexType diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 6c95126390..bcfb5fa7ce 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -40,4 +40,5 @@ engine_config: resource_config: search_resources: # define the device used for search computation - cpu - index_build_device: cpu # CPU used for building index + index_build_device: # CPU used for building index + - cpu diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 154db5d134..a347a9e5bd 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -42,4 +42,5 @@ resource_config: search_resources: # define the devices used for search computation, must be in format: cpu or gpux - cpu - gpu0 - index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu or gpux + index_build_device: # CPU / GPU used for building index, must be in format: cpu or gpux + - gpu0 diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 19c699bda7..dd80ec796b 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -570,12 +570,19 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); - Status s = config.GetResourceConfigIndexBuildDevice(gpu_num_); + std::vector gpu_ids; + Status s = config.GetResourceConfigIndexBuildDevice(gpu_ids); if (!s.ok()) { return s; } + for (auto id : gpu_ids) { + if (gpu_num_ == id) { + return Status::OK(); + } + } - return Status::OK(); + std::string msg = "Invalid gpu_num"; + return Status(SERVER_INVALID_ARGUMENT, msg); } } // namespace engine diff --git a/core/src/scheduler/JobMgr.cpp b/core/src/scheduler/JobMgr.cpp index 794f6a0f37..8dd095a3fa 100644 --- a/core/src/scheduler/JobMgr.cpp +++ b/core/src/scheduler/JobMgr.cpp @@ -104,7 +104,7 @@ JobMgr::build_task(const JobPtr& job) { void JobMgr::calculate_path(const TaskPtr& task) { - if (task->type_ != TaskType::SearchTask) { + if (task->type_ != TaskType::SearchTask && task->type_ != TaskType::BuildIndexTask) { return; } diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 61e0c09759..407c3a44d1 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -54,8 +54,8 @@ load_simple_config() { // get resources auto gpu_ids = get_gpu_pool(); - int32_t build_gpu_id; - config.GetResourceConfigIndexBuildDevice(build_gpu_id); + std::vector build_gpu_ids; + config.GetResourceConfigIndexBuildDevice(build_gpu_ids); // create and connect ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false)); @@ -65,19 +65,30 @@ load_simple_config() { ResMgrInst::GetInstance()->Connect("disk", "cpu", io); auto pcie = Connection("pcie", 12000); - bool find_build_gpu_id = false; - for (auto& gpu_id : gpu_ids) { - ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); - if (build_gpu_id == gpu_id) { - find_build_gpu_id = true; + + std::vector not_find_build_ids; + for (auto& build_id : build_gpu_ids) { + bool find_gpu_id = false; + for (auto& gpu_id : gpu_ids) { + if (gpu_id == build_id) { + find_gpu_id = true; + break; + } + } + if (not find_gpu_id) { + not_find_build_ids.emplace_back(build_id); } } - if (not find_build_gpu_id) { + for (auto& gpu_id : gpu_ids) { + ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); + } + + for (auto& not_find_id : not_find_build_ids) { ResMgrInst::GetInstance()->Add( - ResourceFactory::Create(std::to_string(build_gpu_id), "GPU", build_gpu_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(build_gpu_id), pcie); + ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie); } } diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index a3048069f9..4fd22f1fab 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -21,6 +21,7 @@ #include "JobMgr.h" #include "ResourceMgr.h" #include "Scheduler.h" +#include "optimizer/BuildIndexPass.h" #include "optimizer/HybridPass.h" #include "optimizer/LargeSQ8HPass.h" #include "optimizer/OnlyCPUPass.h" @@ -107,11 +108,15 @@ class OptimizerInst { } } + std::vector build_resources; + config.GetResourceConfigIndexBuildDevice(build_resources); + std::vector pass_list; pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared(has_cpu)); + pass_list.push_back(std::make_shared(build_resources)); instance = std::make_shared(pass_list); } } diff --git a/core/src/scheduler/action/PushTaskToNeighbour.cpp b/core/src/scheduler/action/PushTaskToNeighbour.cpp index b8a4a1164b..9aed678937 100644 --- a/core/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/core/src/scheduler/action/PushTaskToNeighbour.cpp @@ -138,73 +138,41 @@ Action::SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, Resou std::shared_ptr event) { auto task_item = event->task_table_item_; auto task = event->task_table_item_->task; - if (resource->type() == ResourceType::DISK) { - // step 1: calculate shortest path per resource, from disk to compute resource - auto compute_resources = res_mgr->GetComputeResources(); - std::vector> paths; - std::vector transport_costs; - for (auto& res : compute_resources) { - std::vector path; - uint64_t transport_cost = ShortestPath(resource, res, res_mgr, path); - transport_costs.push_back(transport_cost); - paths.emplace_back(path); - } - // if (task->job_.lock()->type() == JobType::SEARCH) { - // auto label = task->label(); - // auto spec_label = std::static_pointer_cast(label); - // if (spec_label->resource().lock()->type() == ResourceType::CPU) { - // std::vector spec_path; - // spec_path.push_back(spec_label->resource().lock()->name()); - // spec_path.push_back(resource->name()); - // task->path() = Path(spec_path, spec_path.size() - 1); - // } else { - // // step 2: select min cost, cost(resource) = avg_cost * task_to_do + transport_cost - // uint64_t min_cost = std::numeric_limits::max(); - // uint64_t min_cost_idx = 0; - // for (uint64_t i = 0; i < compute_resources.size(); ++i) { - // if (compute_resources[i]->TotalTasks() == 0) { - // min_cost_idx = i; - // break; - // } - // uint64_t cost = compute_resources[i]->TaskAvgCost() * - // compute_resources[i]->NumOfTaskToExec() + - // transport_costs[i]; - // if (min_cost > cost) { - // min_cost = cost; - // min_cost_idx = i; - // } - // } - // - // // step 3: set path in task - // Path task_path(paths[min_cost_idx], paths[min_cost_idx].size() - 1); - // task->path() = task_path; - // } - // - // } else - if (task->job_.lock()->type() == JobType::BUILD) { - // step2: Read device id in config - // get build index gpu resource - server::Config& config = server::Config::GetInstance(); - int32_t build_index_gpu; - Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); - - bool find_gpu_res = false; - if (res_mgr->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { - for (uint64_t i = 0; i < compute_resources.size(); ++i) { - if (compute_resources[i]->name() == - res_mgr->GetResource(ResourceType::GPU, build_index_gpu)->name()) { - find_gpu_res = true; - Path task_path(paths[i], paths[i].size() - 1); - task->path() = task_path; - break; - } - } - } - if (not find_gpu_res) { - task->path() = Path(paths[0], paths[0].size() - 1); - } - } - } + // if (resource->type() == ResourceType::DISK) { + // // step 1: calculate shortest path per resource, from disk to compute resource + // auto compute_resources = res_mgr->GetComputeResources(); + // std::vector> paths; + // std::vector transport_costs; + // for (auto& res : compute_resources) { + // std::vector path; + // uint64_t transport_cost = ShortestPath(resource, res, res_mgr, path); + // transport_costs.push_back(transport_cost); + // paths.emplace_back(path); + // } + // if (task->job_.lock()->type() == JobType::BUILD) { + // // step2: Read device id in config + // // get build index gpu resource + // server::Config& config = server::Config::GetInstance(); + // int32_t build_index_gpu; + // Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); + // + // bool find_gpu_res = false; + // if (res_mgr->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { + // for (uint64_t i = 0; i < compute_resources.size(); ++i) { + // if (compute_resources[i]->name() == + // res_mgr->GetResource(ResourceType::GPU, build_index_gpu)->name()) { + // find_gpu_res = true; + // Path task_path(paths[i], paths[i].size() - 1); + // task->path() = task_path; + // break; + // } + // } + // } + // if (not find_gpu_res) { + // task->path() = Path(paths[0], paths[0].size() - 1); + // } + // } + // } if (resource->name() == task->path().Last()) { resource->WakeupExecutor(); diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp new file mode 100644 index 0000000000..725b2509c3 --- /dev/null +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/BuildIndexPass.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/tasklabel/SpecResLabel.h" + +namespace milvus { +namespace scheduler { + +BuildIndexPass::BuildIndexPass(std::vector& build_gpu_ids) : build_gpu_ids_(build_gpu_ids) { +} + +bool +BuildIndexPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::BuildIndexTask) + return false; + + if (build_gpu_ids_.empty()) + return false; + + ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_gpu_ids_[specified_gpu_id_]); + auto label = std::make_shared(std::weak_ptr(res_ptr)); + task->label() = label; + + specified_gpu_id_ = (specified_gpu_id_ + 1) % build_gpu_ids_.size(); + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/BuildIndexPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h new file mode 100644 index 0000000000..a70844bfa9 --- /dev/null +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class BuildIndexPass : public Pass { + public: + explicit BuildIndexPass(std::vector& build_gpu_id); + + public: + bool + Run(const TaskPtr& task) override; + + private: + uint64_t specified_gpu_id_ = 0; + std::vector build_gpu_ids_; +}; + +using BuildIndexPassPtr = std::shared_ptr; + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.cpp b/core/src/scheduler/optimizer/OnlyGPUPass.cpp index 3fcda0e8a3..e867e45159 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.cpp +++ b/core/src/scheduler/optimizer/OnlyGPUPass.cpp @@ -41,12 +41,11 @@ OnlyGPUPass::Run(const TaskPtr& task) { auto gpu_id = get_gpu_pool(); if (gpu_id.empty()) return false; - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpu_id[specified_gpu_id_]); auto label = std::make_shared(std::weak_ptr(res_ptr)); task->label() = label; - specified_gpu_id_ = specified_gpu_id_++ % gpu_id.size(); + specified_gpu_id_ = (specified_gpu_id_ + 1) % gpu_id.size(); return true; } diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index f130e73a85..a2a1354dd4 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -215,8 +215,8 @@ Config::ValidateConfig() { return s; } - int32_t resource_index_build_device; - s = GetResourceConfigIndexBuildDevice(resource_index_build_device); + std::vector index_build_devices; + s = GetResourceConfigIndexBuildDevice(index_build_devices); if (!s.ok()) { return s; } @@ -599,22 +599,24 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } else { uint64_t gpu_cache_capacity = std::stoi(value) * GB; - int device_id; - Status s = GetResourceConfigIndexBuildDevice(device_id); + std::vector device_ids; + Status s = GetResourceConfigIndexBuildDevice(device_ids); if (!s.ok()) { return s; } size_t gpu_memory; - if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { - std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); - return Status(SERVER_UNEXPECTED_ERROR, msg); - } else if (gpu_cache_capacity >= gpu_memory) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { - std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + for (auto& device_id : device_ids) { + if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { + std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); + return Status(SERVER_UNEXPECTED_ERROR, msg); + } else if (gpu_cache_capacity >= gpu_memory) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { + std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + } } } return Status::OK(); @@ -745,11 +747,21 @@ Config::CheckResourceConfigSearchResources(const std::vector& value } Status -Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { - auto status = CheckResource(value); - if (!status.ok()) { - return Status(SERVER_INVALID_ARGUMENT, status.message()); +Config::CheckResourceConfigIndexBuildDevice(const std::vector& value) { + if (value.empty()) { + std::string msg = + "Invalid index build resource. " + "Possible reason: resource_config.index_build_device is empty."; + return Status(SERVER_INVALID_ARGUMENT, msg); } + + for (auto& resource : value) { + auto status = CheckResource(resource); + if (!status.ok()) { + return Status(SERVER_INVALID_ARGUMENT, status.message()); + } + } + return Status::OK(); } @@ -1036,18 +1048,25 @@ Config::GetResourceConfigSearchResources(std::vector& value) { } Status -Config::GetResourceConfigIndexBuildDevice(int32_t& value) { +Config::GetResourceConfigIndexBuildDevice(std::vector& value) { std::string str = - GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); - Status s = CheckResourceConfigIndexBuildDevice(str); + GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DELIMITER, + CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + std::vector resources; + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_INDEX_BUILD_DELIMITER, resources); + + Status s = CheckResourceConfigIndexBuildDevice(resources); if (!s.ok()) { return s; } - if (str == "cpu") { - value = CPU_DEVICE_ID; - } else { - value = std::stoi(str.substr(3)); + for (auto res : resources) { + if (res == "cpu") { + value.emplace_back(CPU_DEVICE_ID); + break; + } + int64_t device_id = std::stoi(str.substr(3)); + value.emplace_back(device_id); } return Status::OK(); @@ -1318,7 +1337,10 @@ Config::SetResourceConfigSearchResources(const std::string& value) { Status Config::SetResourceConfigIndexBuildDevice(const std::string& value) { - Status s = CheckResourceConfigIndexBuildDevice(value); + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_INDEX_BUILD_DELIMITER, res_vec); + + Status s = CheckResourceConfigIndexBuildDevice(res_vec); if (!s.ok()) { return s; } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 3ab0cd8053..08b3accdd6 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -101,10 +101,11 @@ static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0"; #endif static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_DELIMITER = ","; #ifdef MILVUS_CPU_VERSION static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu"; #else -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu,gpu0"; #endif const int32_t CPU_DEVICE_ID = -1; @@ -190,7 +191,7 @@ class Config { Status CheckResourceConfigSearchResources(const std::vector& value); Status - CheckResourceConfigIndexBuildDevice(const std::string& value); + CheckResourceConfigIndexBuildDevice(const std::vector& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -259,7 +260,7 @@ class Config { Status GetResourceConfigSearchResources(std::vector& value); Status - GetResourceConfigIndexBuildDevice(int32_t& value); + GetResourceConfigIndexBuildDevice(std::vector& value); public: /* server config */ diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index 8ed19232e9..ccfbcbb6cf 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -48,12 +48,14 @@ KnowhereResource::Initialize() { // get build index gpu resource server::Config& config = server::Config::GetInstance(); - int32_t build_index_gpu; - s = config.GetResourceConfigIndexBuildDevice(build_index_gpu); + std::vector build_index_gpus; + s = config.GetResourceConfigIndexBuildDevice(build_index_gpus); if (!s.ok()) return s; - gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); + for (auto gpu_id : build_index_gpus) { + gpu_resources.insert(std::make_pair(gpu_id, GpuResourceSetting())); + } // get search gpu resource std::vector pool; diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 637273732d..7f4376bb67 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -292,7 +292,8 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.SetResourceConfigIndexBuildDevice("gpu" + std::to_string(resource_index_build_device)); #endif ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigIndexBuildDevice(int32_val); + std::vector device_ids; + s = config.GetResourceConfigIndexBuildDevice(device_ids); ASSERT_TRUE(s.ok()); ASSERT_TRUE(int32_val == resource_index_build_device); } From ead945870cd8d742f870862e6c60031d22841c0a Mon Sep 17 00:00:00 2001 From: zhenwu Date: Mon, 18 Nov 2019 15:49:37 +0800 Subject: [PATCH 13/56] Update nightly-build config --- ci/jenkins/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index f0562ada64..8d3953b112 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -2,7 +2,7 @@ String cron_timezone = "TZ=Asia/Shanghai" String cron_string = BRANCH_NAME == "master" ? "H 0 * * * " : "" -cron_string = BRANCH_NAME == "0.5.1" ? "H 1 * * * " : cron_string +cron_string = BRANCH_NAME == "0.6.0" ? "H 1 * * * " : cron_string pipeline { agent none From 661a60374e2f9d10cd73b5a676c01cf7bbe181e6 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 18 Nov 2019 15:58:50 +0800 Subject: [PATCH 14/56] dont modify grpc generated code --- core/src/grpc/gen-milvus/milvus.pb.h | 57 ++++++------------- core/src/sdk/grpc/ClientProxy.cpp | 15 ++++- .../grpc_impl/request/SearchRequest.cpp | 2 +- 3 files changed, 29 insertions(+), 45 deletions(-) diff --git a/core/src/grpc/gen-milvus/milvus.pb.h b/core/src/grpc/gen-milvus/milvus.pb.h index 536365b23b..53ed2db22e 100644 --- a/core/src/grpc/gen-milvus/milvus.pb.h +++ b/core/src/grpc/gen-milvus/milvus.pb.h @@ -1321,8 +1321,7 @@ class RowRecord : void clear_vector_data(); float vector_data(int index) const; void set_vector_data(int index, float value); -// void add_vector_data(float value); - void add_vector_data(std::vector::const_iterator begin, std::vector::const_iterator end); + void add_vector_data(float value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& vector_data() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* @@ -1474,9 +1473,7 @@ class InsertParam : void clear_row_id_array(); ::PROTOBUF_NAMESPACE_ID::int64 row_id_array(int index) const; void set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::int64 value); -// void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value); - void add_row_id_array(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end); + void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& row_id_array() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* @@ -2139,9 +2136,7 @@ class TopKQueryResult : void clear_ids(); ::PROTOBUF_NAMESPACE_ID::int64 ids(int index) const; void set_ids(int index, ::PROTOBUF_NAMESPACE_ID::int64 value); -// void add_ids(::PROTOBUF_NAMESPACE_ID::int64 value); - void add_ids(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end); + void add_ids(::PROTOBUF_NAMESPACE_ID::int64 value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& ids() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* @@ -2152,8 +2147,7 @@ class TopKQueryResult : void clear_distances(); float distances(int index) const; void set_distances(int index, float value); -// void add_distances(float value); - void add_distances(std::vector::const_iterator begin, std::vector::const_iterator end); + void add_distances(float value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& distances() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* @@ -3928,14 +3922,9 @@ inline void RowRecord::set_vector_data(int index, float value) { vector_data_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.RowRecord.vector_data) } -//inline void RowRecord::add_vector_data(float value) { -// vector_data_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) -//} -inline void RowRecord::add_vector_data(std::vector::const_iterator begin, - std::vector::const_iterator end) { - vector_data_.Add(begin, end); - // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) +inline void RowRecord::add_vector_data(float value) { + vector_data_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& RowRecord::vector_data() const { @@ -4048,14 +4037,9 @@ inline void InsertParam::set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::in row_id_array_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.InsertParam.row_id_array) } -//inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) { -// row_id_array_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) -//} -inline void InsertParam::add_row_id_array(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end) { - row_id_array_.Add(begin, end); - // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) +inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) { + row_id_array_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& InsertParam::row_id_array() const { @@ -4604,14 +4588,9 @@ inline void TopKQueryResult::set_ids(int index, ::PROTOBUF_NAMESPACE_ID::int64 v ids_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.TopKQueryResult.ids) } -//inline void TopKQueryResult::add_ids(::PROTOBUF_NAMESPACE_ID::int64 value) { -// ids_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) -//} -inline void TopKQueryResult::add_ids(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end) { - ids_.Add(begin,end); - // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) +inline void TopKQueryResult::add_ids(::PROTOBUF_NAMESPACE_ID::int64 value) { + ids_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& TopKQueryResult::ids() const { @@ -4639,13 +4618,9 @@ inline void TopKQueryResult::set_distances(int index, float value) { distances_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.TopKQueryResult.distances) } -//inline void TopKQueryResult::add_distances(float value) { -// distances_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) -//} -inline void TopKQueryResult::add_distances(std::vector::const_iterator begin, std::vector::const_iterator end) { - distances_.Add(begin, end); - // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) +inline void TopKQueryResult::add_distances(float value) { + distances_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& TopKQueryResult::distances() const { diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 131d0bdc99..4a9c319b4d 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -32,6 +32,13 @@ UriCheck(const std::string& uri) { return (index != std::string::npos); } +void +CopyRowRecord(::milvus::grpc::RowRecord* target, const RowRecord& src) { + auto vector_data = target->mutable_vector_data(); + vector_data->Resize(static_cast(src.data.size()), 0.0); + memcpy(vector_data->mutable_data(), src.data.data(), src.data.size() * sizeof(float)); +} + Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; @@ -189,14 +196,16 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_ for (auto& record : record_array) { ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); - grpc_record->add_vector_data(record.data.begin(), record.data.end()); + CopyRowRecord(grpc_record, record); } // Single thread ::milvus::grpc::VectorIds vector_ids; if (!id_array.empty()) { /* set user's ids */ - insert_param.add_row_id_array(id_array.begin(), id_array.end()); + auto row_ids = insert_param.mutable_row_id_array(); + row_ids->Resize(static_cast(id_array.size()), -1); + memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); client_ptr_->Insert(vector_ids, insert_param, status); } else { client_ptr_->Insert(vector_ids, insert_param, status); @@ -226,7 +235,7 @@ ClientProxy::Search(const std::string& table_name, const std::vectoradd_vector_data(record.data.begin(), record.data.end()); + CopyRowRecord(row_record, record); } // step 2: convert range array diff --git a/core/src/server/grpc_impl/request/SearchRequest.cpp b/core/src/server/grpc_impl/request/SearchRequest.cpp index 955576360a..28f4ff723e 100644 --- a/core/src/server/grpc_impl/request/SearchRequest.cpp +++ b/core/src/server/grpc_impl/request/SearchRequest.cpp @@ -167,7 +167,7 @@ SearchRequest::OnExecute() { // step 7: construct result array topk_result_->set_row_num(record_count); - topk_result_->mutable_ids()->Resize(static_cast(result_ids.size()), 0); + topk_result_->mutable_ids()->Resize(static_cast(result_ids.size()), -1); memcpy(topk_result_->mutable_ids()->mutable_data(), result_ids.data(), result_ids.size() * sizeof(int64_t)); topk_result_->mutable_distances()->Resize(static_cast(result_distances.size()), 0.0); memcpy(topk_result_->mutable_distances()->mutable_data(), result_distances.data(), From b36905055bb18d87e12695ed8335b7022f6f85a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=96=E9=BE=99?= Date: Mon, 18 Nov 2019 17:25:38 +0800 Subject: [PATCH 15/56] update add configurable DB settings --- shards/mishards/__init__.py | 4 +++- shards/mishards/db_base.py | 7 ++----- shards/mishards/settings.py | 11 +++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/shards/mishards/__init__.py b/shards/mishards/__init__.py index a3c55c4ae3..55594220d3 100644 --- a/shards/mishards/__init__.py +++ b/shards/mishards/__init__.py @@ -11,7 +11,9 @@ grpc_server = Server() def create_app(testing_config=None): config = testing_config if testing_config else settings.DefaultConfig - db.init_db(uri=config.SQLALCHEMY_DATABASE_URI, echo=config.SQL_ECHO) + db.init_db(uri=config.SQLALCHEMY_DATABASE_URI, echo=config.SQL_ECHO, pool_size=config.SQL_POOL_SIZE, + pool_recycle=config.SQL_POOL_RECYCLE, pool_timeout=config.SQL_POOL_TIMEOUT, + pool_pre_ping=config.SQL_POOL_PRE_PING, max_overflow=config.SQL_MAX_OVERFLOW) from mishards.connections import ConnectionMgr connect_mgr = ConnectionMgr() diff --git a/shards/mishards/db_base.py b/shards/mishards/db_base.py index 5f2eee9ba1..e55c330352 100644 --- a/shards/mishards/db_base.py +++ b/shards/mishards/db_base.py @@ -23,15 +23,12 @@ class DB: uri and self.init_db(uri, echo) self.session_factory = scoped_session(sessionmaker(class_=LocalSession, db=self)) - def init_db(self, uri, echo=False): + def init_db(self, uri, echo=False, pool_size=100, pool_recycle=5, pool_timeout=30, pool_pre_ping=True, max_overflow=0): url = make_url(uri) if url.get_backend_name() == 'sqlite': self.engine = create_engine(url) else: - self.engine = create_engine(uri, pool_size=100, pool_recycle=5, pool_timeout=30, - pool_pre_ping=True, - echo=echo, - max_overflow=0) + self.engine = create_engine(uri, pool_size, pool_recycle, pool_timeout, pool_pre_ping, echo, max_overflow) self.uri = uri self.url = url diff --git a/shards/mishards/settings.py b/shards/mishards/settings.py index 8d7361dddc..832f1639ea 100644 --- a/shards/mishards/settings.py +++ b/shards/mishards/settings.py @@ -50,10 +50,16 @@ class TracingConfig: } } + max_overflow=0 class DefaultConfig: SQLALCHEMY_DATABASE_URI = env.str('SQLALCHEMY_DATABASE_URI') SQL_ECHO = env.bool('SQL_ECHO', False) + SQL_POOL_SIZE = env.int('pool_size', 100) + SQL_POOL_RECYCLE = env.int('pool_recycle', 5) + SQL_POOL_TIMEOUT = env.int('pool_timeout', 30) + SQL_POOL_PRE_PING = env.bool('pool_pre_ping', True) + SQL_MAX_OVERFLOW = env.int('max_overflow', 0) TRACER_PLUGIN_PATH = env.str('TRACER_PLUGIN_PATH', '') TRACER_CLASS_NAME = env.str('TRACER_CLASS_NAME', '') ROUTER_PLUGIN_PATH = env.str('ROUTER_PLUGIN_PATH', '') @@ -65,5 +71,10 @@ class DefaultConfig: class TestingConfig(DefaultConfig): SQLALCHEMY_DATABASE_URI = env.str('SQLALCHEMY_DATABASE_TEST_URI', '') SQL_ECHO = env.bool('SQL_TEST_ECHO', False) + SQL_POOL_SIZE = env.int('pool_size', 100) + SQL_POOL_RECYCLE = env.int('pool_recycle', 5) + SQL_POOL_TIMEOUT = env.int('pool_timeout', 30) + SQL_POOL_PRE_PING = env.bool('pool_pre_ping', True) + SQL_MAX_OVERFLOW = env.int('max_overflow', 0) TRACER_CLASS_NAME = env.str('TRACER_CLASS_TEST_NAME', '') ROUTER_CLASS_NAME = env.str('ROUTER_CLASS_TEST_NAME', 'FileBasedHashRingRouter') From a5f4484f9d760385a3f972c84eb4377c670fd9ec Mon Sep 17 00:00:00 2001 From: "G.Y Feng" <40229765+GuanyunFeng@users.noreply.github.com> Date: Mon, 18 Nov 2019 18:31:12 +0800 Subject: [PATCH 16/56] Update README_CN.md Add a link to C++ SDK in README_CN #383 --- README_CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_CN.md b/README_CN.md index 5dad64af9b..8f4d4b05aa 100644 --- a/README_CN.md +++ b/README_CN.md @@ -20,7 +20,7 @@ Milvus 是一款开源的、针对海量特征向量的相似性搜索引擎。基于异构众核计算框架设计，成本更低，性能更好。在有限的计算资源下，十亿向量搜索仅毫秒响应。 -Milvus 提供稳定的 Python、Java 以及 C++ 的 API 接口。 +Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java) 以及 [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) 的 API 接口。 通过 [版本发布说明](https://milvus.io/docs/zh-CN/release/v0.5.0/) 获取最新发行版本的 Milvus。 From 039f23e08205831017022d2dcab560d5f279431c Mon Sep 17 00:00:00 2001 From: "G.Y Feng" <40229765+GuanyunFeng@users.noreply.github.com> Date: Mon, 18 Nov 2019 18:32:32 +0800 Subject: [PATCH 17/56] Update README.md Add a link to C++ SDK in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 311d0d6843..4a1151dbf8 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Milvus is the world's fastest similarity search engine for massive-scale feature For more detailed introduction of Milvus and its architecture, see [Milvus overview](https://www.milvus.io/docs/en/aboutmilvus/overview/). -Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and C++ APIs. +Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) APIs. Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://www.milvus.io/docs/en/release/v0.5.0/). From 7e67c76e2c5c9a5791de18e9d5ed131f03ef1681 Mon Sep 17 00:00:00 2001 From: "G.Y Feng" <40229765+GuanyunFeng@users.noreply.github.com> Date: Mon, 18 Nov 2019 18:34:31 +0800 Subject: [PATCH 18/56] Update ubuntu_build_deps.sh solve issue #387, "failed to create symbolic link '/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so': File exists" --- core/ubuntu_build_deps.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/ubuntu_build_deps.sh b/core/ubuntu_build_deps.sh index 911046aa1f..6f3fabc084 100755 --- a/core/ubuntu_build_deps.sh +++ b/core/ubuntu_build_deps.sh @@ -9,4 +9,6 @@ sudo apt-get -y update && sudo apt-get -y install intel-mkl-gnu-2019.5-281 intel sudo apt-get install -y gfortran libmysqlclient-dev mysql-client libcurl4-openssl-dev libboost-system-dev \ libboost-filesystem-dev libboost-serialization-dev libboost-regex-dev -sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so +if [ ! -f "/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so" ]; then + sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so +fi From fddfd1eb03c067234e6127730da14677c03c7554 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Mon, 18 Nov 2019 18:47:00 +0800 Subject: [PATCH 19/56] #346 update config APIs to support build index with multiple GPUs --- core/conf/server_cpu_config.template | 6 +- core/conf/server_gpu_config.template | 3 +- core/src/server/Config.cpp | 84 +++++++++++++++------------- core/src/server/Config.h | 16 +++--- core/src/utils/ValidationUtil.cpp | 4 +- core/src/utils/ValidationUtil.h | 4 +- core/unittest/server/test_config.cpp | 33 ++++++----- 7 files changed, 81 insertions(+), 69 deletions(-) diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 6c95126390..bc8fc3bb3d 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -27,7 +27,6 @@ metric_config: port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] cache_config: - cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] cache_insert_data: false # whether to load inserted data into cache, must be a boolean @@ -38,6 +37,7 @@ engine_config: gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only resource_config: - search_resources: # define the device used for search computation + search_resources: # define the devices used for search computation, must be in format: cpu or gpux + - cpu + index_build_resources: # define the devices used for index building, must be in format: cpu or gpux - cpu - index_build_device: cpu # CPU used for building index diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 154db5d134..c54ed408df 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -42,4 +42,5 @@ resource_config: search_resources: # define the devices used for search computation, must be in format: cpu or gpux - cpu - gpu0 - index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu or gpux + index_build_resources: # define the devices used for index building, must be in format: cpu or gpux + - gpu0 \ No newline at end of file diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index f130e73a85..5672ab52aa 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -215,8 +215,8 @@ Config::ValidateConfig() { return s; } - int32_t resource_index_build_device; - s = GetResourceConfigIndexBuildDevice(resource_index_build_device); + std::vector index_build_resources; + s = GetResourceConfigIndexBuildResources(index_build_resources); if (!s.ok()) { return s; } @@ -351,7 +351,7 @@ Config::ResetDefaultConfig() { return s; } - s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + s = SetResourceConfigIndexBuildResources(CONFIG_RESOURCE_INDEX_BUILD_RESOURCES_DEFAULT); if (!s.ok()) { return s; } @@ -599,22 +599,28 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } else { uint64_t gpu_cache_capacity = std::stoi(value) * GB; - int device_id; - Status s = GetResourceConfigIndexBuildDevice(device_id); + std::vector resources; + Status s = GetResourceConfigIndexBuildResources(resources); if (!s.ok()) { return s; } size_t gpu_memory; - if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { - std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); - return Status(SERVER_UNEXPECTED_ERROR, msg); - } else if (gpu_cache_capacity >= gpu_memory) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { - std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + for (auto& resource : resources) { + if (resource == "cpu") { + continue; + } + int32_t device_id = std::stoi(resource.substr(3)); + if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { + std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); + return Status(SERVER_UNEXPECTED_ERROR, msg); + } else if (gpu_cache_capacity >= gpu_memory) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else if (gpu_cache_capacity > (double) gpu_memory * 0.9) { + std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + } } } return Status::OK(); @@ -745,10 +751,18 @@ Config::CheckResourceConfigSearchResources(const std::vector& value } Status -Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { - auto status = CheckResource(value); - if (!status.ok()) { - return Status(SERVER_INVALID_ARGUMENT, status.message()); +Config::CheckResourceConfigIndexBuildResources(const std::vector& value) { + if (value.empty()) { + std::string msg = + "Invalid build index resource. " + "Possible reason: resource_config.build_index_resources is empty."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + for (auto& resource : value) { + auto status = CheckResource(resource); + if (!status.ok()) { + return Status(SERVER_INVALID_ARGUMENT, status.message()); + } } return Status::OK(); } @@ -1030,27 +1044,18 @@ Status Config::GetResourceConfigSearchResources(std::vector& value) { std::string str = GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, - CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); - server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value); + CONFIG_RESOURCE_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_RESOURCES_DELIMITER, value); return CheckResourceConfigSearchResources(value); } Status -Config::GetResourceConfigIndexBuildDevice(int32_t& value) { +Config::GetResourceConfigIndexBuildResources(std::vector& value) { std::string str = - GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); - Status s = CheckResourceConfigIndexBuildDevice(str); - if (!s.ok()) { - return s; - } - - if (str == "cpu") { - value = CPU_DEVICE_ID; - } else { - value = std::stoi(str.substr(3)); - } - - return Status::OK(); + GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_RESOURCES, + CONFIG_RESOURCE_RESOURCES_DELIMITER, CONFIG_RESOURCE_INDEX_BUILD_RESOURCES_DEFAULT); + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_RESOURCES_DELIMITER, value); + return CheckResourceConfigIndexBuildResources(value); } /////////////////////////////////////////////////////////////////////////////// @@ -1305,7 +1310,7 @@ Config::SetResourceConfigMode(const std::string& value) { Status Config::SetResourceConfigSearchResources(const std::string& value) { std::vector res_vec; - server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec); + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_RESOURCES_DELIMITER, res_vec); Status s = CheckResourceConfigSearchResources(res_vec); if (!s.ok()) { @@ -1317,13 +1322,16 @@ Config::SetResourceConfigSearchResources(const std::string& value) { } Status -Config::SetResourceConfigIndexBuildDevice(const std::string& value) { - Status s = CheckResourceConfigIndexBuildDevice(value); +Config::SetResourceConfigIndexBuildResources(const std::string &value) { + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_RESOURCES_DELIMITER, res_vec); + + Status s = CheckResourceConfigIndexBuildResources(res_vec); if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value); + SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_RESOURCES, value); return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 3ab0cd8053..0378a079fb 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -91,20 +91,18 @@ static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; static const char* CONFIG_RESOURCE = "resource_config"; static const char* CONFIG_RESOURCE_MODE = "mode"; static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; +static const char* CONFIG_RESOURCE_RESOURCES_DELIMITER = ","; static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources"; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ","; - #ifdef MILVUS_CPU_VERSION static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu"; #else static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0"; #endif - -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_RESOURCES = "index_build_resources"; #ifdef MILVUS_CPU_VERSION -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_RESOURCES_DEFAULT = "cpu"; #else -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_RESOURCES_DEFAULT = "gpu0"; #endif const int32_t CPU_DEVICE_ID = -1; @@ -190,7 +188,7 @@ class Config { Status CheckResourceConfigSearchResources(const std::vector& value); Status - CheckResourceConfigIndexBuildDevice(const std::string& value); + CheckResourceConfigIndexBuildResources(const std::vector& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -259,7 +257,7 @@ class Config { Status GetResourceConfigSearchResources(std::vector& value); Status - GetResourceConfigIndexBuildDevice(int32_t& value); + GetResourceConfigIndexBuildResources(std::vector& value); public: /* server config */ @@ -320,7 +318,7 @@ class Config { Status SetResourceConfigSearchResources(const std::string& value); Status - SetResourceConfigIndexBuildDevice(const std::string& value); + SetResourceConfigIndexBuildResources(const std::string& value); private: std::unordered_map> config_map_; diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index ec696ff3e0..080de77e17 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -182,7 +182,7 @@ ValidationUtil::ValidatePartitionTags(const std::vector& partition_ } Status -ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { +ValidationUtil::ValidateGpuIndex(int32_t gpu_index) { #ifdef MILVUS_GPU_VERSION int num_devices = 0; auto cuda_err = cudaGetDeviceCount(&num_devices); @@ -203,7 +203,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { } Status -ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { +ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) { #ifdef MILVUS_GPU_VERSION cudaDeviceProp deviceProp; diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 01801e295a..201ccef3bd 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -59,10 +59,10 @@ class ValidationUtil { ValidatePartitionTags(const std::vector& partition_tags); static Status - ValidateGpuIndex(uint32_t gpu_index); + ValidateGpuIndex(int32_t gpu_index); static Status - GetGpuMemory(uint32_t gpu_index, size_t& memory); + GetGpuMemory(int32_t gpu_index, size_t& memory); static Status ValidateIpAddress(const std::string& ip_address); diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 637273732d..37be36b7eb 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -272,29 +272,34 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { #else std::vector search_resources = {"cpu", "gpu0"}; #endif - std::vector res_vec; - std::string res_str; + std::vector search_res_vec; + std::string search_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( - search_resources, milvus::server::CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_str); - s = config.SetResourceConfigSearchResources(res_str); + search_resources, milvus::server::CONFIG_RESOURCE_RESOURCES_DELIMITER, search_res_str); + s = config.SetResourceConfigSearchResources(search_res_str); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigSearchResources(res_vec); + s = config.GetResourceConfigSearchResources(search_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < search_resources.size(); i++) { - ASSERT_TRUE(search_resources[i] == res_vec[i]); + ASSERT_TRUE(search_resources[i] == search_res_vec[i]); } #ifdef MILVUS_CPU_VERSION - int32_t resource_index_build_device = milvus::server::CPU_DEVICE_ID; - s = config.SetResourceConfigIndexBuildDevice("cpu"); + std::vector index_build_resources = {"cpu"}; #else - int32_t resource_index_build_device = 0; - s = config.SetResourceConfigIndexBuildDevice("gpu" + std::to_string(resource_index_build_device)); + std::vector index_build_resources = {"gpu0", "gpu1"}; #endif + std::vector index_build_res_vec; + std::string index_build_res_str; + milvus::server::StringHelpFunctions::MergeStringWithDelimeter( + index_build_resources, milvus::server::CONFIG_RESOURCE_RESOURCES_DELIMITER, index_build_res_str); + s = config.SetResourceConfigIndexBuildResources(index_build_res_str); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigIndexBuildDevice(int32_val); + s = config.GetResourceConfigIndexBuildResources(index_build_res_vec); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == resource_index_build_device); + for (size_t i = 0; i < index_build_resources.size(); i++) { + ASSERT_TRUE(index_build_resources[i] == index_build_res_vec[i]); + } } TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { @@ -418,9 +423,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetResourceConfigSearchResources("cpu"); ASSERT_TRUE(s.ok()); - s = config.SetResourceConfigIndexBuildDevice("gup2"); + s = config.SetResourceConfigIndexBuildResources("gup2"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigIndexBuildDevice("gpu16"); + s = config.SetResourceConfigIndexBuildResources("gpu16"); ASSERT_FALSE(s.ok()); } From 3eefd550591b092db769464f801824ab231c3751 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 18 Nov 2019 18:54:25 +0800 Subject: [PATCH 20/56] #397 sdk_simple return incorrect result --- CHANGELOG.md | 1 + core/src/sdk/examples/utils/Utils.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9240b357a0..eb4fba56c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#340 - Test cases run failed on 0.6.0 - \#353 - Rename config.h.in to version.h.in - \#374 - sdk_simple return empty result +- \#397 - sdk_simple return incorrect result ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/sdk/examples/utils/Utils.cpp b/core/src/sdk/examples/utils/Utils.cpp index d9dd950710..da5e854e9b 100644 --- a/core/src/sdk/examples/utils/Utils.cpp +++ b/core/src/sdk/examples/utils/Utils.cpp @@ -167,7 +167,7 @@ Utils::PrintSearchResult(const std::vector index++; std::cout << "No." << index << " vector " << search_id << " top " << topk << " search result:" << std::endl; for (size_t j = 0; j < topk; j++) { - size_t idx = i * nq + j; + size_t idx = i * topk + j; std::cout << "\t" << topk_query_result.ids[idx] << "\t" << topk_query_result.distances[idx] << std::endl; } } From 55ada7533c3918aa4e2480428aa1f71744043a5d Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Mon, 18 Nov 2019 19:10:37 +0800 Subject: [PATCH 21/56] #255 - Add ivfsq8 test report detailed version --- ...vus_ivfsq8_test_report_detailed_version.md | 210 +++++++++++++++++ ..._ivfsq8_test_report_detailed_version_cn.md | 211 ++++++++++++++++++ 2 files changed, 421 insertions(+) create mode 100644 docs/test_report/milvus_ivfsq8_test_report_detailed_version.md create mode 100644 docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md new file mode 100644 index 0000000000..4d6840430e --- /dev/null +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md @@ -0,0 +1,210 @@ +# milvus_ivfsq8_test_report_detailed_version + +## Summary + +This document contains the test reports of IVF_SQ8 index on Milvus single server. + + + +## Test objectives + +The time cost and recall when searching with different parameters. + + + +## Test method + +### Hardware/Software requirements + +Operating System: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +Memory: 503GB + +Docker version: 18.09 + +Nvidia Driver version: 430.34 + +Milvus version: 0.5.3 + +SDK interface: Python 3.6.8 + +Pymilvus version: 0.2.5 + + + +### Data model + +The data used in the tests are: + +- Data source: sift1b +- Data type: hdf5 + +For details on this dataset, you can check : http://corpus-texmex.irisa.fr/ . + + + +### Measures + +- Query Elapsed Time: Time cost (in seconds) to run a query. Variables that affect Query Elapsed Time: + + - nq (Number of queried vectors) + + > Note: In the query test of query elapsed time, we will test the following parameters with different values: + > + > nq - grouped by: [1, 5, 10, 200, 400, 600, 800, 1000], + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved . Variables that affect Recall: + + - nq (Number of queried vectors) + - topk (Top k result of a query) + + > Note: In the query test of recall, we will test the following parameters with different values: + > + > nq - grouped by: [1, 5, 10, 200, 400, 600, 800, 1000], + > + > topk - grouped by: [1, 10, 100] + + + +## Test reports + +### Test environment + +Data base: sift1b-1,000,000,000 vectors, 128-dimension + +Table Attributes + +- nlist: 16384 +- metric_type: L2 + +Query configuration + +- nprobe: 32 + +Milvus configuration + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 + +You can check the definition of Milvus configuration on https://milvus.io/docs/en/reference/milvus_config/. + +Test method + +Test the query elapsed time and recall with several parameters, and once only change one parameter. + +- Whether to restart Milvus after each query: No + + + +### Performance test + +#### Data query + +**Test result** + +Query Elapsed Time + +topk : 100 + +search_resources: gpu0, gpu1 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 15.57 | +| nq=10 | 15.80 | +| nq=200 | 15.72 | +| nq=400 | 15.94 | +| nq=600 | 16.58 | +| nq=800 | 16.71 | +| nq=1000 | 16.91 | + +When nq is 1000, the query time cost of a 128-dimension vector is around 17ms in GPU Mode. + + + +topk : 100 + +search_resources: cpu, gpu0 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 1.12 | +| nq=10 | 2.89 | +| nq=200 | 8.10 | +| nq=400 | 12.36 | +| nq=600 | 17.81 | +| nq=800 | 23.24 | +| nq=1000 | 27.41 | + +When nq is 1000, the query time cost of a 128-dimension vector is around 27ms in CPU Mode. + + + +**Conclusion** + +The query elapsed time in CPU Mode increases quickly with nq, while in GPU Mode query elapsed time increases much slower. When nq is small, CPU Mode consumes less time than GPU Mode. However, as nq becomes larger, GPU Mode shows its advantage against CPU Mode. + +The query elapsed time in GPU Mode consists of two parts: (1) index CPU-to-GPU copy time; (2) nprobe buckets search time. When nq is smaller than 500, index CPU-to-GPU copy time cannot be amortized efficiently, CPU Mode is a better choice; when nq is larger than 500, choosing GPU Mode is better. + +Compared with CPU, GPU has much more cores and stronger computing capability. When nq is large, it can better reflect GPU's advantages on computing. + + + +### Recall test + +**Test result** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +We use the ground_truth in sift1b dataset to calculate the recall of query results. + + + +Recall of GPU Mode + +search_resources: gpu0, gpu1 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=1 | 1.000 | 0.800 | 0.790 | +| nq=5 | 0.800 | 0.820 | 0.908 | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +Recall of CPU Mode + +search_resources: cpu, gpu0 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=1 | 1.000 | 0.800 | 0.790 | +| nq=5 | 0.800 | 0.820 | 0.908 | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +**Conclusion** + +As nq increases, the recall gradually stabilizes to over 93%. \ No newline at end of file diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md new file mode 100644 index 0000000000..4b5e18f291 --- /dev/null +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md @@ -0,0 +1,211 @@ +# milvus_ivfsq8_test_report_detailed_version_cn + +## 概述 + +本文描述了ivfsq8索引在milvus单机部署方式下的测试报告。 + + + +## 测试目标 + +参数不同情况下的查询时间和召回率。 + + + +## 测试方法 + +### 软硬件环境 + +操作系统: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +内存: 503GB + +Docker版本: 18.09 + +Nvidia Driver版本: 430.34 + +Milvus版本: 0.5.3 + +SDK接口: Python 3.6.8 + +Pymilvus版本: 0.2.5 + + + +### 数据模型 + +本测试中用到的主要数据: + +- 数据来源: sift1b +- 数据类型: hdf5 + +关于该数据集的详细信息请参考 : http://corpus-texmex.irisa.fr/ 。 + + + +### 测试指标 + +- Query Elapsed Time: 数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量: + + - nq (被查询向量的数量) + + > 备注：在向量查询测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [1, 5, 10, 200, 400, 600, 800, 1000]的数量分组。 + +- Recall: 实际返回的正确结果占总数之比 . 影响Recall的变量: + + - nq (被查询向量的数量) + - topk (单条查询中最相似的K个结果) + + > 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [1, 5, 10, 200, 400, 600, 800, 1000]的数量分组， + > + > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。 + + + +## 测试报告 + +### 测试环境 + +数据集: sift1b-1,000,000,000向量, 128维 + +表格属性： + +- nlist: 16384 +- metric_type: L2 + +查询设置： + +- nprobe: 32 + +Milvus设置： + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 + +你可以在 https://milvus.io/docs/en/reference/milvus_config/上查询Milvus设置的详细定义。 + +测试方法 + +通过一次仅改变一个参数的值，测试查询向量时间和召回率。 + +- 查询后是否重启Milvus：否 + + + +### 性能测试 + +#### 数据查询 + +测试结果 + +Query Elapsed Time + +topk : 100 + +search_resources: gpu0, gpu1 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 15.57 | +| nq=10 | 15.80 | +| nq=200 | 15.72 | +| nq=400 | 15.94 | +| nq=600 | 16.58 | +| nq=800 | 16.71 | +| nq=1000 | 16.91 | + +当nq为1000时，在GPU模式下查询一条128维向量需要耗时约17毫秒。 + + + +topk : 100 + +search_resources: cpu, gpu0 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 1.12 | +| nq=10 | 2.89 | +| nq=200 | 8.10 | +| nq=400 | 12.36 | +| nq=600 | 17.81 | +| nq=800 | 23.24 | +| nq=1000 | 27.41 | + +当nq为1000时，在GPU模式下查询一条128维向量需要耗时约27毫秒。 + + + +**总结** + +在CPU模式下查询耗时随nq的增长快速增大，而在GPU模式下查询耗时的增大则缓慢许多。当nq较小时，CPU模式比GPU模式耗时更少。但当nq足够大时，GPU模式则更具有优势。 + +在GPU模式下的查询耗时由两部分组成：（1）索引从CPU到GPU的拷贝时间；（2）所有分桶的查询时间。当nq小于500时，索引从CPU到GPU 的拷贝时间无法被有效均摊，此时CPU模式时一个更优的选择；当nq大于500时，选择GPU模式更合理。 + +和CPU相比，GPU具有更多的核数和更强的算力。当nq较大时，GPU在计算上的优势能被更好地被体现。 + + + +### 召回率测试 + +**测试结果** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +我们利用sift1b数据集中的ground_truth来计算查询结果的召回率。 + + + +Recall of GPU Mode + +search_resources: gpu0, gpu1 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=1 | 1.000 | 0.800 | 0.790 | +| nq=5 | 0.800 | 0.820 | 0.908 | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +Recall of CPU Mode + +search_resources: cpu, gpu0 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=1 | 1.000 | 0.800 | 0.790 | +| nq=5 | 0.800 | 0.820 | 0.908 | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +**总结** + +​ 随着nq的增大，召回率逐渐稳定至93%以上。 + From e70c8e029ef4534f15e5c0e18e302c4aa17e76f1 Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Mon, 18 Nov 2019 19:12:36 +0800 Subject: [PATCH 22/56] #255 - Add ivfsq8 test report detailed version, update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9240b357a0..4bcbb0b10a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md +- \#255 - Add ivfsq8 test report detailed version ## Task From 562b84a178afb90344e26bda3c021bff80e5f60c Mon Sep 17 00:00:00 2001 From: Shouyu Luo Date: Mon, 18 Nov 2019 23:03:59 +0800 Subject: [PATCH 23/56] commented out logs --- core/src/db/DBImpl.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index fc31846bd3..52cf1e668a 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -71,12 +71,12 @@ DBImpl::Start() { return Status::OK(); } - ENGINE_LOG_TRACE << "DB service start"; + // ENGINE_LOG_TRACE << "DB service start"; shutting_down_.store(false, std::memory_order_release); // for distribute version, some nodes are read only if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { - ENGINE_LOG_TRACE << "StartTimerTasks"; + // ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); } @@ -101,7 +101,7 @@ DBImpl::Stop() { meta_ptr_->CleanUp(); } - ENGINE_LOG_TRACE << "DB service stop"; + // ENGINE_LOG_TRACE << "DB service stop"; return Status::OK(); } @@ -506,7 +506,7 @@ DBImpl::StartMetricTask() { return; } - ENGINE_LOG_TRACE << "Start metric task"; + // ENGINE_LOG_TRACE << "Start metric task"; server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); @@ -532,7 +532,7 @@ DBImpl::StartMetricTask() { server::Metrics::GetInstance().GPUTemperature(); server::Metrics::GetInstance().CPUTemperature(); - ENGINE_LOG_TRACE << "Metric task finished"; + // ENGINE_LOG_TRACE << "Metric task finished"; } Status @@ -692,7 +692,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) { void DBImpl::BackgroundCompaction(std::set table_ids) { - ENGINE_LOG_TRACE << " Background compaction thread start"; + // ENGINE_LOG_TRACE << " Background compaction thread start"; Status status; for (auto& table_id : table_ids) { @@ -715,7 +715,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { } meta_ptr_->CleanUpFilesWithTTL(ttl); - ENGINE_LOG_TRACE << " Background compaction thread exit"; + // ENGINE_LOG_TRACE << " Background compaction thread exit"; } void @@ -748,7 +748,7 @@ DBImpl::StartBuildIndexTask(bool force) { void DBImpl::BackgroundBuildIndex() { - ENGINE_LOG_TRACE << "Background build index thread start"; + // ENGINE_LOG_TRACE << "Background build index thread start"; std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; @@ -771,7 +771,7 @@ DBImpl::BackgroundBuildIndex() { } } - ENGINE_LOG_TRACE << "Background build index thread exit"; + // ENGINE_LOG_TRACE << "Background build index thread exit"; } } // namespace engine From 8f64c44271785753f2fb34b34edb08598a950ba8 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Tue, 19 Nov 2019 09:16:58 +0800 Subject: [PATCH 24/56] [skip ci] Add note about using unbuntu docker --- install.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install.md b/install.md index 6711b41f76..98b9af4649 100644 --- a/install.md +++ b/install.md @@ -3,6 +3,9 @@ ## Software requirements - Ubuntu 18.04 or higher + + If your Ubuntu version is lower than 18.04, we recommend you to pull a [docker image of Ubuntu 18.04](https://docs.docker.com/install/linux/docker-ce/ubuntu/) as your compilation environment. + - CMake 3.12 or higher ##### For GPU version, you will also need: From 9fb536b502e6d17f1606c117b117bf55e1442a8b Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Tue, 19 Nov 2019 11:49:05 +0800 Subject: [PATCH 25/56] [skip ci] minor update --- install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.md b/install.md index 98b9af4649..f53586af36 100644 --- a/install.md +++ b/install.md @@ -4,7 +4,7 @@ - Ubuntu 18.04 or higher - If your Ubuntu version is lower than 18.04, we recommend you to pull a [docker image of Ubuntu 18.04](https://docs.docker.com/install/linux/docker-ce/ubuntu/) as your compilation environment. + If your operating system is not Ubuntu 18.04 or higher, we recommend you to pull a [docker image of Ubuntu 18.04](https://docs.docker.com/install/linux/docker-ce/ubuntu/) as your compilation environment. - CMake 3.12 or higher From 7ad5c514970497a4583098f07756d7c25176eea8 Mon Sep 17 00:00:00 2001 From: wxyu Date: Tue, 19 Nov 2019 12:29:05 +0800 Subject: [PATCH 26/56] Add virtual method Init() in Pass abstract class close #404 --- CHANGELOG.md | 1 + core/src/scheduler/SchedInst.cpp | 1 + core/src/scheduler/optimizer/HybridPass.cpp | 4 ++++ core/src/scheduler/optimizer/HybridPass.h | 3 +++ core/src/scheduler/optimizer/LargeSQ8HPass.cpp | 3 ++- core/src/scheduler/optimizer/LargeSQ8HPass.h | 5 ++++- core/src/scheduler/optimizer/OnlyCPUPass.cpp | 4 ++++ core/src/scheduler/optimizer/OnlyCPUPass.h | 3 +++ core/src/scheduler/optimizer/OnlyGPUPass.cpp | 4 ++++ core/src/scheduler/optimizer/OnlyGPUPass.h | 3 +++ core/src/scheduler/optimizer/Optimizer.cpp | 12 ++++++------ core/src/scheduler/optimizer/Optimizer.h | 4 ++-- core/src/scheduler/optimizer/Pass.h | 5 ++--- 13 files changed, 39 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bcbb0b10a..a87e4bec63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md - \#255 - Add ivfsq8 test report detailed version +- \#404 - Add virtual method Init() in Pass abstract class ## Task diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 61e0c09759..9bd4df944f 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -84,6 +84,7 @@ load_simple_config() { void StartSchedulerService() { load_simple_config(); + OptimizerInst::GetInstance()->Init(); ResMgrInst::GetInstance()->Start(); SchedInst::GetInstance()->Start(); JobMgrInst::GetInstance()->Start(); diff --git a/core/src/scheduler/optimizer/HybridPass.cpp b/core/src/scheduler/optimizer/HybridPass.cpp index d63fc2e819..8bf0d8990d 100644 --- a/core/src/scheduler/optimizer/HybridPass.cpp +++ b/core/src/scheduler/optimizer/HybridPass.cpp @@ -23,6 +23,10 @@ namespace milvus { namespace scheduler { +void +HybridPass::Init() { +} + bool HybridPass::Run(const TaskPtr& task) { // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu diff --git a/core/src/scheduler/optimizer/HybridPass.h b/core/src/scheduler/optimizer/HybridPass.h index 0d02a8bda9..f84a0884f2 100644 --- a/core/src/scheduler/optimizer/HybridPass.h +++ b/core/src/scheduler/optimizer/HybridPass.h @@ -37,6 +37,9 @@ class HybridPass : public Pass { HybridPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; }; diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp index b9784e3c0a..89bd85ec1f 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp @@ -27,7 +27,8 @@ namespace milvus { namespace scheduler { -LargeSQ8HPass::LargeSQ8HPass() { +void +LargeSQ8HPass::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/LargeSQ8HPass.h index 9d135d413a..1afd240c0f 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.h +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.h @@ -35,9 +35,12 @@ namespace scheduler { class LargeSQ8HPass : public Pass { public: - LargeSQ8HPass(); + LargeSQ8HPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.cpp b/core/src/scheduler/optimizer/OnlyCPUPass.cpp index 238a91a82c..392ae7940b 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.cpp +++ b/core/src/scheduler/optimizer/OnlyCPUPass.cpp @@ -24,6 +24,10 @@ namespace milvus { namespace scheduler { +void +OnlyCPUPass::Init() { +} + bool OnlyCPUPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::SearchTask) diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.h b/core/src/scheduler/optimizer/OnlyCPUPass.h index 76b42e3766..253775f77e 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.h +++ b/core/src/scheduler/optimizer/OnlyCPUPass.h @@ -37,6 +37,9 @@ class OnlyCPUPass : public Pass { OnlyCPUPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; }; diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.cpp b/core/src/scheduler/optimizer/OnlyGPUPass.cpp index 2a72f9757e..69a8366bb2 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.cpp +++ b/core/src/scheduler/optimizer/OnlyGPUPass.cpp @@ -27,6 +27,10 @@ namespace scheduler { OnlyGPUPass::OnlyGPUPass(bool has_cpu) : has_cpu_(has_cpu) { } +void +OnlyGPUPass::Init() { +} + bool OnlyGPUPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::SearchTask || has_cpu_) diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.h b/core/src/scheduler/optimizer/OnlyGPUPass.h index 10d909d30e..c7a83fea30 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.h +++ b/core/src/scheduler/optimizer/OnlyGPUPass.h @@ -37,6 +37,9 @@ class OnlyGPUPass : public Pass { explicit OnlyGPUPass(bool has_cpu); public: + void + Init() override; + bool Run(const TaskPtr& task) override; diff --git a/core/src/scheduler/optimizer/Optimizer.cpp b/core/src/scheduler/optimizer/Optimizer.cpp index 46f24ea712..c5fa311a27 100644 --- a/core/src/scheduler/optimizer/Optimizer.cpp +++ b/core/src/scheduler/optimizer/Optimizer.cpp @@ -20,12 +20,12 @@ namespace milvus { namespace scheduler { -// void -// Optimizer::Init() { -// for (auto& pass : pass_list_) { -// pass->Init(); -// } -// } +void +Optimizer::Init() { + for (auto& pass : pass_list_) { + pass->Init(); + } +} bool Optimizer::Run(const TaskPtr& task) { diff --git a/core/src/scheduler/optimizer/Optimizer.h b/core/src/scheduler/optimizer/Optimizer.h index bfabbf7de3..68b519e115 100644 --- a/core/src/scheduler/optimizer/Optimizer.h +++ b/core/src/scheduler/optimizer/Optimizer.h @@ -38,8 +38,8 @@ class Optimizer { explicit Optimizer(std::vector pass_list) : pass_list_(std::move(pass_list)) { } - // void - // Init(); + void + Init(); bool Run(const TaskPtr& task); diff --git a/core/src/scheduler/optimizer/Pass.h b/core/src/scheduler/optimizer/Pass.h index 016b05e457..36a36a1df5 100644 --- a/core/src/scheduler/optimizer/Pass.h +++ b/core/src/scheduler/optimizer/Pass.h @@ -34,9 +34,8 @@ namespace scheduler { class Pass { public: - // virtual void - // Init() { - // } + virtual void + Init() = 0; virtual bool Run(const TaskPtr& task) = 0; From 31aa89e1318c3e8b9b2e1cee220b20b7fc42370b Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Tue, 19 Nov 2019 12:59:54 +0800 Subject: [PATCH 27/56] Change ivfsq8 test report detailed version --- .../milvus_ivfsq8_test_report_detailed_version.md | 4 ---- .../milvus_ivfsq8_test_report_detailed_version_cn.md | 6 +----- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md index 4d6840430e..dc1bbb3fd0 100644 --- a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md @@ -177,8 +177,6 @@ search_resources: gpu0, gpu1 | nq/topk | topk=1 | topk=10 | topk=100 | | :-----: | :----: | :-----: | :------: | -| nq=1 | 1.000 | 0.800 | 0.790 | -| nq=5 | 0.800 | 0.820 | 0.908 | | nq=10 | 0.900 | 0.910 | 0.939 | | nq=200 | 0.955 | 0.941 | 0.929 | | nq=400 | 0.958 | 0.944 | 0.932 | @@ -194,8 +192,6 @@ search_resources: cpu, gpu0 | nq/topk | topk=1 | topk=10 | topk=100 | | :-----: | :----: | :-----: | :------: | -| nq=1 | 1.000 | 0.800 | 0.790 | -| nq=5 | 0.800 | 0.820 | 0.908 | | nq=10 | 0.900 | 0.910 | 0.939 | | nq=200 | 0.955 | 0.941 | 0.929 | | nq=400 | 0.958 | 0.944 | 0.932 | diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md index 4b5e18f291..bf43d2e0e8 100644 --- a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md @@ -177,8 +177,6 @@ search_resources: gpu0, gpu1 | nq/topk | topk=1 | topk=10 | topk=100 | | :-----: | :----: | :-----: | :------: | -| nq=1 | 1.000 | 0.800 | 0.790 | -| nq=5 | 0.800 | 0.820 | 0.908 | | nq=10 | 0.900 | 0.910 | 0.939 | | nq=200 | 0.955 | 0.941 | 0.929 | | nq=400 | 0.958 | 0.944 | 0.932 | @@ -194,8 +192,6 @@ search_resources: cpu, gpu0 | nq/topk | topk=1 | topk=10 | topk=100 | | :-----: | :----: | :-----: | :------: | -| nq=1 | 1.000 | 0.800 | 0.790 | -| nq=5 | 0.800 | 0.820 | 0.908 | | nq=10 | 0.900 | 0.910 | 0.939 | | nq=200 | 0.955 | 0.941 | 0.929 | | nq=400 | 0.958 | 0.944 | 0.932 | @@ -207,5 +203,5 @@ search_resources: cpu, gpu0 **总结** -​ 随着nq的增大，召回率逐渐稳定至93%以上。 +随着nq的增大，召回率逐渐稳定至93%以上。 From 2707e07ccd36e47d1ff60d3be9e95832a6942b28 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 19 Nov 2019 14:31:32 +0800 Subject: [PATCH 28/56] Fix Deployed GPU-version docker failed in CI --- ci/jenkins/step/deploySingle2Dev.groovy | 6 +----- ci/jenkins/step/singleDevNightlyTest.groovy | 6 +----- ci/jenkins/step/singleDevTest.groovy | 6 +----- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index f1daaf22ec..7b479ff44a 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,11 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - if ("${env.BINRARY_VERSION}" == "gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } else { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/filebeat/values.yaml --namespace milvus ." - } + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index d357badfd3..e27bc6c0a5 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,11 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - if ("${env.BINRARY_VERSION}" == "gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } else { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index c1de5907b0..08812ae01f 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,11 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // if ("${env.BINRARY_VERSION}" == "gpu") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - // } else { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - // } + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { From 0d1923c61f53ea6ec77c98356e6c8301317d9b74 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Tue, 19 Nov 2019 15:49:40 +0800 Subject: [PATCH 29/56] #346 update gpu resource config APIs --- core/conf/server_cpu_config.template | 15 +- core/conf/server_gpu_config.template | 15 +- core/src/cache/GpuCacheMgr.cpp | 4 +- core/src/server/Config.cpp | 462 ++++++++++++++------------- core/src/server/Config.h | 87 +++-- core/src/utils/ValidationUtil.cpp | 4 +- core/src/utils/ValidationUtil.h | 4 +- core/unittest/server/test_config.cpp | 116 ++++--- 8 files changed, 357 insertions(+), 350 deletions(-) diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 6c95126390..ae942f351e 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -27,9 +27,7 @@ metric_config: port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] cache_config: - - cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] + cpu_cache_capacity: 16 # GB, size of CPU memory used for cache, must be a positive integer cache_insert_data: false # whether to load inserted data into cache, must be a boolean engine_config: @@ -37,7 +35,10 @@ engine_config: # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only -resource_config: - search_resources: # define the device used for search computation - - cpu - index_build_device: cpu # CPU used for building index +gpu_resource_config: + enable_gpu: true # whether to enable GPU resources + cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer + search_resources: # define the GPU devices used for search computation, must be in format gpux + - gpu0 + build_index_resources: # define the GPU devices used for index building, must be in format gpux + - gpu0 \ No newline at end of file diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 154db5d134..92a294d483 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -27,10 +27,7 @@ metric_config: port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] cache_config: - cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] - gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer - gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] + cpu_cache_capacity: 16 # GB, size of CPU memory used for cache, must be a positive integer cache_insert_data: false # whether to load inserted data into cache, must be a boolean engine_config: @@ -38,8 +35,10 @@ engine_config: # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only -resource_config: - search_resources: # define the devices used for search computation, must be in format: cpu or gpux - - cpu +gpu_resource_config: + enable_gpu: false # whether to enable GPU resources + cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer + search_resources: # define the GPU devices used for search computation, must be in format gpux - gpu0 - index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu or gpux + build_index_resources: # define the GPU devices used for index building, must be in format gpux + - gpu0 \ No newline at end of file diff --git a/core/src/cache/GpuCacheMgr.cpp b/core/src/cache/GpuCacheMgr.cpp index d862bc0393..72229527fa 100644 --- a/core/src/cache/GpuCacheMgr.cpp +++ b/core/src/cache/GpuCacheMgr.cpp @@ -37,7 +37,7 @@ GpuCacheMgr::GpuCacheMgr() { Status s; int64_t gpu_cache_cap; - s = config.GetCacheConfigGpuCacheCapacity(gpu_cache_cap); + s = config.GetGpuResourceConfigCacheCapacity(gpu_cache_cap); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } @@ -45,7 +45,7 @@ GpuCacheMgr::GpuCacheMgr() { cache_ = std::make_shared>(cap, 1UL << 32); float gpu_mem_threshold; - s = config.GetCacheConfigGpuCacheThreshold(gpu_mem_threshold); + s = config.GetGpuResourceConfigCacheThreshold(gpu_mem_threshold); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index f130e73a85..017f1641bd 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -163,20 +163,6 @@ Config::ValidateConfig() { return s; } -#ifdef MILVUS_GPU_VERSION - int64_t cache_gpu_cache_capacity; - s = GetCacheConfigGpuCacheCapacity(cache_gpu_cache_capacity); - if (!s.ok()) { - return s; - } - - float cache_gpu_cache_threshold; - s = GetCacheConfigGpuCacheThreshold(cache_gpu_cache_threshold); - if (!s.ok()) { - return s; - } -#endif - bool cache_insert_data; s = GetCacheConfigCacheInsertData(cache_insert_data); if (!s.ok()) { @@ -202,25 +188,39 @@ Config::ValidateConfig() { return s; } - /* resource config */ - std::string resource_mode; - s = GetResourceConfigMode(resource_mode); + /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION + bool resource_enable_gpu; + s = GetGpuResourceConfigEnableGpu(resource_enable_gpu); if (!s.ok()) { return s; } - std::vector search_resources; - s = GetResourceConfigSearchResources(search_resources); + int64_t resource_cache_capacity; + s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity); if (!s.ok()) { return s; } - int32_t resource_index_build_device; - s = GetResourceConfigIndexBuildDevice(resource_index_build_device); + float resource_cache_threshold; + s = GetGpuResourceConfigCacheThreshold(resource_cache_threshold); if (!s.ok()) { return s; } + std::vector search_resources; + s = GetGpuResourceConfigSearchResources(search_resources); + if (!s.ok()) { + return s; + } + + std::vector index_build_resources; + s = GetGpuResourceConfigBuildIndexResources(index_build_resources); + if (!s.ok()) { + return s; + } +#endif + return Status::OK(); } @@ -307,18 +307,6 @@ Config::ResetDefaultConfig() { return s; } -#ifdef MILVUS_GPU_VERSION - s = SetCacheConfigGpuCacheCapacity(CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetCacheConfigGpuCacheThreshold(CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); - if (!s.ok()) { - return s; - } -#endif - s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); if (!s.ok()) { return s; @@ -340,22 +328,34 @@ Config::ResetDefaultConfig() { return s; } - /* resource config */ - s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT); + /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION + s = SetGpuResourceConfigEnableGpu(CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT); if (!s.ok()) { return s; } - s = SetResourceConfigSearchResources(CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); + s = SetGpuResourceConfigCacheCapacity(CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); if (!s.ok()) { return s; } - s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + s = SetGpuResourceConfigCacheThreshold(CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT); if (!s.ok()) { return s; } + s = SetGpuResourceConfigSearchResources(CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT); + if (!s.ok()) { + return s; + } + + s = SetGpuResourceConfigBuildIndexResources(CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT); + if (!s.ok()) { + return s; + } +#endif + return Status::OK(); } @@ -377,7 +377,7 @@ Config::PrintAll() { PrintConfigSection(CONFIG_CACHE); PrintConfigSection(CONFIG_METRIC); PrintConfigSection(CONFIG_ENGINE); - PrintConfigSection(CONFIG_RESOURCE); + PrintConfigSection(CONFIG_GPU_RESOURCE); } //////////////////////////////////////////////////////////////////////////////// @@ -591,52 +591,6 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { return Status::OK(); } -Status -Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - uint64_t gpu_cache_capacity = std::stoi(value) * GB; - int device_id; - Status s = GetResourceConfigIndexBuildDevice(device_id); - if (!s.ok()) { - return s; - } - - size_t gpu_memory; - if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { - std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); - return Status(SERVER_UNEXPECTED_ERROR, msg); - } else if (gpu_cache_capacity >= gpu_memory) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { - std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; - } - } - return Status::OK(); -} - -Status -Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) { - if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { - std::string msg = "Invalid gpu cache threshold: " + value + - ". Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - float gpu_cache_threshold = std::stof(value); - if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { - std::string msg = "Invalid gpu cache threshold: " + value + - ". Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } - return Status::OK(); -} - Status Config::CheckCacheConfigCacheInsertData(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { @@ -687,56 +641,99 @@ Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { } Status -Config::CheckResourceConfigMode(const std::string& value) { - if (value != "simple") { - std::string msg = "Invalid resource mode: " + value + ". Possible reason: resource_config.mode is invalid."; +Config::CheckGpuResourceConfigEnableGpu(const std::string& value) { + if (!ValidationUtil::ValidateStringIsBool(value).ok()) { + std::string msg = "Invalid gpu resource config: " + value + + ". Possible reason: gpu_resource_config.enable_gpu is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } Status -CheckResource(const std::string& value) { +Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: gpu_resource_config.cache_capacity is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + uint64_t gpu_cache_capacity = std::stoi(value) * GB; + std::vector gpu_ids; + Status s = GetGpuResourceConfigBuildIndexResources(gpu_ids); + if (!s.ok()) { + return s; + } + + for (int32_t gpu_id : gpu_ids) { + size_t gpu_memory; + if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) { + std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id); + return Status(SERVER_UNEXPECTED_ERROR, msg); + } else if (gpu_cache_capacity >= gpu_memory) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: gpu_resource_config.cache_capacity exceeds GPU memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { + std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + } + } + } + return Status::OK(); +} + +Status +Config::CheckGpuResourceConfigCacheThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { + std::string msg = "Invalid gpu cache threshold: " + value + + ". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + float gpu_cache_threshold = std::stof(value); + if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { + std::string msg = "Invalid gpu cache threshold: " + value + + ". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + } + return Status::OK(); +} + +Status +CheckGpuResource(const std::string& value) { std::string s = value; std::transform(s.begin(), s.end(), s.begin(), ::tolower); -#ifdef MILVUS_CPU_VERSION - if (s != "cpu") { - return Status(SERVER_INVALID_ARGUMENT, "Invalid CPU resource: " + s); - } -#else - const std::regex pat("cpu|gpu(\\d+)"); + const std::regex pat("gpu(\\d+)"); std::smatch m; if (!std::regex_match(s, m, pat)) { - std::string msg = "Invalid search resource: " + value + - ". Possible reason: resource_config.search_resources is not in the format of cpux or gpux"; + std::string msg = "Invalid gpu resource: " + value + + ". Possible reason: gpu_resource_config is not in the format of cpux or gpux"; return Status(SERVER_INVALID_ARGUMENT, msg); } if (s.compare(0, 3, "gpu") == 0) { int32_t gpu_index = std::stoi(s.substr(3)); if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - std::string msg = "Invalid search resource: " + value + - ". Possible reason: resource_config.search_resources does not match your hardware."; + std::string msg = "Invalid gpu resource: " + value + + ". Possible reason: gpu_resource_config does not match with the hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } } -#endif return Status::OK(); } Status -Config::CheckResourceConfigSearchResources(const std::vector& value) { +Config::CheckGpuResourceConfigSearchResources(const std::vector& value) { if (value.empty()) { std::string msg = - "Invalid search resource. " - "Possible reason: resource_config.search_resources is empty."; + "Invalid gpu search resource. " + "Possible reason: gpu_resource_config.search_resources is empty."; return Status(SERVER_INVALID_ARGUMENT, msg); } for (auto& resource : value) { - auto status = CheckResource(resource); + auto status = CheckGpuResource(resource); if (!status.ok()) { return Status(SERVER_INVALID_ARGUMENT, status.message()); } @@ -745,10 +742,19 @@ Config::CheckResourceConfigSearchResources(const std::vector& value } Status -Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { - auto status = CheckResource(value); - if (!status.ok()) { - return Status(SERVER_INVALID_ARGUMENT, status.message()); +Config::CheckGpuResourceConfigBuildIndexResources(const std::vector& value) { + if (value.empty()) { + std::string msg = + "Invalid gpu build index resource. " + "Possible reason: gpu_resource_config.build_index_resources is empty."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + + for (auto& resource : value) { + auto status = CheckGpuResource(resource); + if (!status.ok()) { + return Status(SERVER_INVALID_ARGUMENT, status.message()); + } } return Status::OK(); } @@ -855,7 +861,6 @@ Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } @@ -868,7 +873,6 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } @@ -880,7 +884,6 @@ Config::GetDBConfigInsertBufferSize(int32_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } @@ -898,7 +901,6 @@ Config::GetMetricConfigEnableMonitor(bool& value) { if (!s.ok()) { return s; } - std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); @@ -924,7 +926,6 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } @@ -937,33 +938,6 @@ Config::GetCacheConfigCpuCacheThreshold(float& value) { if (!s.ok()) { return s; } - - value = std::stof(str); - return Status::OK(); -} - -Status -Config::GetCacheConfigGpuCacheCapacity(int64_t& value) { - std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); - Status s = CheckCacheConfigGpuCacheCapacity(str); - if (!s.ok()) { - return s; - } - - value = std::stoi(str); - return Status::OK(); -} - -Status -Config::GetCacheConfigGpuCacheThreshold(float& value) { - std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); - Status s = CheckCacheConfigGpuCacheThreshold(str); - if (!s.ok()) { - return s; - } - value = std::stof(str); return Status::OK(); } @@ -976,7 +950,6 @@ Config::GetCacheConfigCacheInsertData(bool& value) { if (!s.ok()) { return s; } - std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); @@ -990,7 +963,6 @@ Config::GetEngineConfigUseBlasThreshold(int32_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } @@ -1002,7 +974,6 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } @@ -1015,41 +986,113 @@ Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } Status -Config::GetResourceConfigMode(std::string& value) { - value = GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - return CheckResourceConfigMode(value); -} - -Status -Config::GetResourceConfigSearchResources(std::vector& value) { +Config::GetGpuResourceConfigEnableGpu(bool& value) { std::string str = - GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, - CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); - server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value); - return CheckResourceConfigSearchResources(value); -} - -Status -Config::GetResourceConfigIndexBuildDevice(int32_t& value) { - std::string str = - GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); - Status s = CheckResourceConfigIndexBuildDevice(str); + GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE_GPU, CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT); + Status s = CheckGpuResourceConfigEnableGpu(str); if (!s.ok()) { return s; } + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + value = (str == "true" || str == "on" || str == "yes" || str == "1"); + return Status::OK(); +} - if (str == "cpu") { - value = CPU_DEVICE_ID; - } else { - value = std::stoi(str.substr(3)); +Status +Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { + bool enable_gpu = false; + Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + if (!s.ok()) { + return s; } + if (!enable_gpu) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, + CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); + s = CheckGpuResourceConfigCacheCapacity(str); + if (!s.ok()) { + return s; + } + value = std::stoi(str); + return Status::OK(); +} +Status +Config::GetGpuResourceConfigCacheThreshold(float& value) { + bool enable_gpu = false; + Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + if (!s.ok()) { + return s; + } + if (!enable_gpu) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, + CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT); + s = CheckGpuResourceConfigCacheThreshold(str); + if (!s.ok()) { + return s; + } + value = std::stof(str); + return Status::OK(); +} + +Status +Config::GetGpuResourceConfigSearchResources(std::vector& value) { + bool enable_gpu = false; + Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + if (!s.ok()) { + return s; + } + if (!enable_gpu) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, + CONFIG_GPU_RESOURCE_DELIMITER, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT); + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + s = CheckGpuResourceConfigSearchResources(res_vec); + if (!s.ok()) { + return s; + } + for (std::string& res : res_vec) { + value.push_back(std::stoi(res.substr(3))); + } + return Status::OK(); +} + +Status +Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { + bool enable_gpu = false; + Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + if (!s.ok()) { + return s; + } + if (!enable_gpu) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = + GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, + CONFIG_GPU_RESOURCE_DELIMITER, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT); + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + s = CheckGpuResourceConfigBuildIndexResources(res_vec); + if (!s.ok()) { + return s; + } + for (std::string& res : res_vec) { + value.push_back(std::stoi(res.substr(3))); + } return Status::OK(); } @@ -1061,7 +1104,6 @@ Config::SetServerConfigAddress(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); return Status::OK(); } @@ -1072,7 +1114,6 @@ Config::SetServerConfigPort(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); return Status::OK(); } @@ -1083,7 +1124,6 @@ Config::SetServerConfigDeployMode(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); return Status::OK(); } @@ -1094,7 +1134,6 @@ Config::SetServerConfigTimeZone(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); return Status::OK(); } @@ -1106,7 +1145,6 @@ Config::SetDBConfigPrimaryPath(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); return Status::OK(); } @@ -1117,7 +1155,6 @@ Config::SetDBConfigSecondaryPath(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); return Status::OK(); } @@ -1128,7 +1165,6 @@ Config::SetDBConfigBackendUrl(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); return Status::OK(); } @@ -1139,7 +1175,6 @@ Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); return Status::OK(); } @@ -1150,7 +1185,6 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); return Status::OK(); } @@ -1161,7 +1195,6 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); return Status::OK(); } @@ -1173,7 +1206,6 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); return Status::OK(); } @@ -1184,7 +1216,6 @@ Config::SetMetricConfigCollector(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); return Status::OK(); } @@ -1195,7 +1226,6 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); return Status::OK(); } @@ -1207,7 +1237,6 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); return Status::OK(); } @@ -1218,40 +1247,16 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); return Status::OK(); } -Status -Config::SetCacheConfigGpuCacheCapacity(const std::string& value) { - Status s = CheckCacheConfigGpuCacheCapacity(value); - if (!s.ok()) { - return s; - } - - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); - return Status::OK(); -} - -Status -Config::SetCacheConfigGpuCacheThreshold(const std::string& value) { - Status s = CheckCacheConfigGpuCacheThreshold(value); - if (!s.ok()) { - return s; - } - - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); - return Status::OK(); -} - Status Config::SetCacheConfigCacheInsertData(const std::string& value) { Status s = CheckCacheConfigCacheInsertData(value); if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); return Status::OK(); } @@ -1263,7 +1268,6 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); return Status::OK(); } @@ -1274,7 +1278,6 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); return Status::OK(); } @@ -1285,45 +1288,62 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value); return Status::OK(); } -/* resource config */ +/* gpu resource config */ Status -Config::SetResourceConfigMode(const std::string& value) { - Status s = CheckResourceConfigMode(value); +Config::SetGpuResourceConfigEnableGpu(const std::string& value) { + Status s = CheckGpuResourceConfigEnableGpu(value); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE_GPU, value); return Status::OK(); } Status -Config::SetResourceConfigSearchResources(const std::string& value) { +Config::SetGpuResourceConfigCacheCapacity(const std::string& value) { + Status s = CheckGpuResourceConfigCacheCapacity(value); + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, value); + return Status::OK(); +} + +Status +Config::SetGpuResourceConfigCacheThreshold(const std::string& value) { + Status s = CheckGpuResourceConfigCacheThreshold(value); + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetGpuResourceConfigSearchResources(const std::string& value) { std::vector res_vec; - server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec); - - Status s = CheckResourceConfigSearchResources(res_vec); + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + Status s = CheckGpuResourceConfigSearchResources(res_vec); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, value); return Status::OK(); } Status -Config::SetResourceConfigIndexBuildDevice(const std::string& value) { - Status s = CheckResourceConfigIndexBuildDevice(value); +Config::SetGpuResourceConfigBuildIndexResources(const std::string& value) { + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + Status s = CheckGpuResourceConfigBuildIndexResources(res_vec); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, value); return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 3ab0cd8053..0442ae0626 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -59,12 +59,8 @@ static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; static const char* CONFIG_CACHE = "cache_config"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "16"; -static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; -static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT = "4"; -static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_mem_threshold"; +static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_cache_threshold"; static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.85"; -static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD = "gpu_mem_threshold"; -static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; @@ -87,26 +83,23 @@ static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD = "gpu_search_threshold"; static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; -/* resource config */ -static const char* CONFIG_RESOURCE = "resource_config"; -static const char* CONFIG_RESOURCE_MODE = "mode"; -static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources"; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ","; - -#ifdef MILVUS_CPU_VERSION -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu"; +/* gpu resource config */ +static const char* CONFIG_GPU_RESOURCE = "gpu_resource_config"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU = "enable_gpu"; +#ifdef MILVUS_GPU_VERSION +static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "true"; #else -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "false"; #endif - -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device"; -#ifdef MILVUS_CPU_VERSION -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu"; -#else -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0"; -#endif -const int32_t CPU_DEVICE_ID = -1; +static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY = "cache_capacity"; +static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "4"; +static const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD = "cache_threshold"; +static const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_GPU_RESOURCE_DELIMITER = ","; +static const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES = "search_resources"; +static const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT = "gpu0"; +static const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES = "build_index_resources"; +static const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT = "gpu0"; class Config { public: @@ -170,10 +163,6 @@ class Config { Status CheckCacheConfigCpuCacheThreshold(const std::string& value); Status - CheckCacheConfigGpuCacheCapacity(const std::string& value); - Status - CheckCacheConfigGpuCacheThreshold(const std::string& value); - Status CheckCacheConfigCacheInsertData(const std::string& value); /* engine config */ @@ -184,13 +173,17 @@ class Config { Status CheckEngineConfigGpuSearchThreshold(const std::string& value); - /* resource config */ + /* gpu resource config */ Status - CheckResourceConfigMode(const std::string& value); + CheckGpuResourceConfigEnableGpu(const std::string& value); Status - CheckResourceConfigSearchResources(const std::vector& value); + CheckGpuResourceConfigCacheCapacity(const std::string& value); Status - CheckResourceConfigIndexBuildDevice(const std::string& value); + CheckGpuResourceConfigCacheThreshold(const std::string& value); + Status + CheckGpuResourceConfigSearchResources(const std::vector& value); + Status + CheckGpuResourceConfigBuildIndexResources(const std::vector& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -239,10 +232,6 @@ class Config { Status GetCacheConfigCpuCacheThreshold(float& value); Status - GetCacheConfigGpuCacheCapacity(int64_t& value); - Status - GetCacheConfigGpuCacheThreshold(float& value); - Status GetCacheConfigCacheInsertData(bool& value); /* engine config */ @@ -253,13 +242,17 @@ class Config { Status GetEngineConfigGpuSearchThreshold(int32_t& value); - /* resource config */ + /* gpu resource config */ Status - GetResourceConfigMode(std::string& value); + GetGpuResourceConfigEnableGpu(bool& value); Status - GetResourceConfigSearchResources(std::vector& value); + GetGpuResourceConfigCacheCapacity(int64_t& value); Status - GetResourceConfigIndexBuildDevice(int32_t& value); + GetGpuResourceConfigCacheThreshold(float& value); + Status + GetGpuResourceConfigSearchResources(std::vector& value); + Status + GetGpuResourceConfigBuildIndexResources(std::vector& value); public: /* server config */ @@ -300,10 +293,6 @@ class Config { Status SetCacheConfigCpuCacheThreshold(const std::string& value); Status - SetCacheConfigGpuCacheCapacity(const std::string& value); - Status - SetCacheConfigGpuCacheThreshold(const std::string& value); - Status SetCacheConfigCacheInsertData(const std::string& value); /* engine config */ @@ -314,13 +303,17 @@ class Config { Status SetEngineConfigGpuSearchThreshold(const std::string& value); - /* resource config */ + /* gpu resource config */ Status - SetResourceConfigMode(const std::string& value); + SetGpuResourceConfigEnableGpu(const std::string& value); Status - SetResourceConfigSearchResources(const std::string& value); + SetGpuResourceConfigCacheCapacity(const std::string& value); Status - SetResourceConfigIndexBuildDevice(const std::string& value); + SetGpuResourceConfigCacheThreshold(const std::string& value); + Status + SetGpuResourceConfigSearchResources(const std::string& value); + Status + SetGpuResourceConfigBuildIndexResources(const std::string& value); private: std::unordered_map> config_map_; diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index ec696ff3e0..080de77e17 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -182,7 +182,7 @@ ValidationUtil::ValidatePartitionTags(const std::vector& partition_ } Status -ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { +ValidationUtil::ValidateGpuIndex(int32_t gpu_index) { #ifdef MILVUS_GPU_VERSION int num_devices = 0; auto cuda_err = cudaGetDeviceCount(&num_devices); @@ -203,7 +203,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { } Status -ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { +ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) { #ifdef MILVUS_GPU_VERSION cudaDeviceProp deviceProp; diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 01801e295a..201ccef3bd 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -59,10 +59,10 @@ class ValidationUtil { ValidatePartitionTags(const std::vector& partition_tags); static Status - ValidateGpuIndex(uint32_t gpu_index); + ValidateGpuIndex(int32_t gpu_index); static Status - GetGpuMemory(uint32_t gpu_index, size_t& memory); + GetGpuMemory(int32_t gpu_index, size_t& memory); static Status ValidateIpAddress(const std::string& ip_address); diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 637273732d..387d321b85 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -216,21 +216,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.GetCacheConfigCpuCacheThreshold(float_val); ASSERT_TRUE(float_val == cache_cpu_cache_threshold); -#ifdef MILVUS_GPU_VERSION - int64_t cache_gpu_cache_capacity = 1; - s = config.SetCacheConfigGpuCacheCapacity(std::to_string(cache_gpu_cache_capacity)); - ASSERT_TRUE(s.ok()); - s = config.GetCacheConfigGpuCacheCapacity(int64_val); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int64_val == cache_gpu_cache_capacity); - - float cache_gpu_cache_threshold = 0.2; - s = config.SetCacheConfigGpuCacheThreshold(std::to_string(cache_gpu_cache_threshold)); - ASSERT_TRUE(s.ok()); - s = config.GetCacheConfigGpuCacheThreshold(float_val); - ASSERT_TRUE(float_val == cache_gpu_cache_threshold); -#endif - bool cache_insert_data = true; s = config.SetCacheConfigCacheInsertData(std::to_string(cache_insert_data)); ASSERT_TRUE(s.ok()); @@ -259,42 +244,54 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(s.ok()); ASSERT_TRUE(int32_val == engine_gpu_search_threshold); - /* resource config */ - std::string resource_mode = "simple"; - s = config.SetResourceConfigMode(resource_mode); + /* gpu resource config */ + bool resource_enable_gpu = true; + s = config.SetGpuResourceConfigEnableGpu(std::to_string(resource_enable_gpu)); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigMode(str_val); + s = config.GetGpuResourceConfigEnableGpu(bool_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(str_val == resource_mode); + ASSERT_TRUE(bool_val == resource_enable_gpu); -#ifdef MILVUS_CPU_VERSION - std::vector search_resources = {"cpu"}; -#else - std::vector search_resources = {"cpu", "gpu0"}; -#endif - std::vector res_vec; - std::string res_str; - milvus::server::StringHelpFunctions::MergeStringWithDelimeter( - search_resources, milvus::server::CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_str); - s = config.SetResourceConfigSearchResources(res_str); +#ifdef MILVUS_GPU_VERSION + int64_t gpu_cache_capacity = 1; + s = config.SetGpuResourceConfigCacheCapacity(std::to_string(gpu_cache_capacity)); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigSearchResources(res_vec); + s = config.GetGpuResourceConfigCacheCapacity(int64_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int64_val == gpu_cache_capacity); + + float gpu_cache_threshold = 0.2; + s = config.SetGpuResourceConfigCacheThreshold(std::to_string(gpu_cache_threshold)); + ASSERT_TRUE(s.ok()); + s = config.GetGpuResourceConfigCacheThreshold(float_val); + ASSERT_TRUE(float_val == gpu_cache_threshold); + + std::vector search_resources = {"gpu0"}; + std::vector search_res_vec; + std::string search_res_str; + milvus::server::StringHelpFunctions::MergeStringWithDelimeter( + search_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, search_res_str); + s = config.SetGpuResourceConfigSearchResources(search_res_str); + ASSERT_TRUE(s.ok()); + s = config.GetGpuResourceConfigSearchResources(search_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < search_resources.size(); i++) { - ASSERT_TRUE(search_resources[i] == res_vec[i]); + ASSERT_TRUE(std::stoi(search_resources[i].substr(3)) == search_res_vec[i]); } -#ifdef MILVUS_CPU_VERSION - int32_t resource_index_build_device = milvus::server::CPU_DEVICE_ID; - s = config.SetResourceConfigIndexBuildDevice("cpu"); -#else - int32_t resource_index_build_device = 0; - s = config.SetResourceConfigIndexBuildDevice("gpu" + std::to_string(resource_index_build_device)); + std::vector build_index_resources = {"gpu0"}; + std::vector build_index_res_vec; + std::string build_index_res_str; + milvus::server::StringHelpFunctions::MergeStringWithDelimeter( + build_index_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, build_index_res_str); + s = config.SetGpuResourceConfigBuildIndexResources(build_index_res_str); + ASSERT_TRUE(s.ok()); + s = config.GetGpuResourceConfigBuildIndexResources(build_index_res_vec); + ASSERT_TRUE(s.ok()); + for (size_t i = 0; i < build_index_resources.size(); i++) { + ASSERT_TRUE(std::stoi(build_index_resources[i].substr(3)) == build_index_res_vec[i]); + } #endif - ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigIndexBuildDevice(int32_val); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == resource_index_build_device); } TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { @@ -381,18 +378,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetCacheConfigCpuCacheThreshold("1.0"); ASSERT_FALSE(s.ok()); -#ifdef MILVUS_GPU_VERSION - s = config.SetCacheConfigGpuCacheCapacity("a"); - ASSERT_FALSE(s.ok()); - s = config.SetCacheConfigGpuCacheCapacity("128"); - ASSERT_FALSE(s.ok()); - - s = config.SetCacheConfigGpuCacheThreshold("a"); - ASSERT_FALSE(s.ok()); - s = config.SetCacheConfigGpuCacheThreshold("1.0"); - ASSERT_FALSE(s.ok()); -#endif - s = config.SetCacheConfigCacheInsertData("N"); ASSERT_FALSE(s.ok()); @@ -408,20 +393,29 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetEngineConfigGpuSearchThreshold("-1"); ASSERT_FALSE(s.ok()); - /* resource config */ - s = config.SetResourceConfigMode("default"); + /* gpu resource config */ + s = config.SetGpuResourceConfigEnableGpu("ok"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigSearchResources("gpu10"); +#ifdef MILVUS_GPU_VERSION + s = config.SetGpuResourceConfigCacheCapacity("a"); + ASSERT_FALSE(s.ok()); + s = config.SetGpuResourceConfigCacheCapacity("128"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigSearchResources("cpu"); - ASSERT_TRUE(s.ok()); + s = config.SetGpuResourceConfigCacheThreshold("a"); + ASSERT_FALSE(s.ok()); + s = config.SetGpuResourceConfigCacheThreshold("1.0"); + ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigIndexBuildDevice("gup2"); + s = config.SetGpuResourceConfigSearchResources("gpu10"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigIndexBuildDevice("gpu16"); + + s = config.SetGpuResourceConfigBuildIndexResources("gup2"); ASSERT_FALSE(s.ok()); + s = config.SetGpuResourceConfigBuildIndexResources("gpu16"); + ASSERT_FALSE(s.ok()); +#endif } TEST_F(ConfigTest, SERVER_CONFIG_TEST) { From 9f063aa2893e6009d7b89d1fad75ca61c24dbba5 Mon Sep 17 00:00:00 2001 From: wxyu Date: Tue, 19 Nov 2019 16:53:04 +0800 Subject: [PATCH 30/56] Add a Fallback pass in optimizer close #409 add Fallback pass condition --- CHANGELOG.md | 1 + core/src/scheduler/SchedInst.h | 2 + core/src/scheduler/optimizer/FallbackPass.cpp | 43 +++++++++++++++++++ core/src/scheduler/optimizer/FallbackPass.h | 40 +++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 core/src/scheduler/optimizer/FallbackPass.cpp create mode 100644 core/src/scheduler/optimizer/FallbackPass.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 98bab925da..ced0633cdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#358 - Add more information in build.sh and install.md - \#255 - Add ivfsq8 test report detailed version - \#404 - Add virtual method Init() in Pass abstract class +- \#409 - Add a Fallback pass in optimizer ## Task diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index a3048069f9..ad02d32081 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -21,6 +21,7 @@ #include "JobMgr.h" #include "ResourceMgr.h" #include "Scheduler.h" +#include "optimizer/FallbackPass.h" #include "optimizer/HybridPass.h" #include "optimizer/LargeSQ8HPass.h" #include "optimizer/OnlyCPUPass.h" @@ -112,6 +113,7 @@ class OptimizerInst { pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared(has_cpu)); + pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); } } diff --git a/core/src/scheduler/optimizer/FallbackPass.cpp b/core/src/scheduler/optimizer/FallbackPass.cpp new file mode 100644 index 0000000000..2e275ede4b --- /dev/null +++ b/core/src/scheduler/optimizer/FallbackPass.cpp @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FallbackPass.h" +#include "scheduler/SchedInst.h" +#include "scheduler/tasklabel/SpecResLabel.h" + +namespace milvus { +namespace scheduler { + +void +FallbackPass::Init() { +} + +bool +FallbackPass::Run(const TaskPtr& task) { + auto task_type = task->Type(); + if (task_type != TaskType::SearchTask && task_type != TaskType::BuildIndexTask) { + return false; + } + // NEVER be empty + auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0]; + auto label = std::make_shared(cpu); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/FallbackPass.h b/core/src/scheduler/optimizer/FallbackPass.h new file mode 100644 index 0000000000..728740d53a --- /dev/null +++ b/core/src/scheduler/optimizer/FallbackPass.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class FallbackPass : public Pass { + public: + FallbackPass() = default; + + public: + void + Init() override; + + bool + Run(const TaskPtr& task) override; +}; + +} // namespace scheduler +} // namespace milvus From 0011765b2db28ca0550a5f1ae31c1299a3cc7799 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Tue, 19 Nov 2019 17:30:29 +0800 Subject: [PATCH 31/56] Change ivfsq8 test report detailed version #255 --- .../milvus_ivfsq8_test_report_detailed_version.md | 8 ++++---- .../milvus_ivfsq8_test_report_detailed_version_cn.md | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md index dc1bbb3fd0..7bea764df0 100644 --- a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md @@ -28,13 +28,13 @@ Memory: 503GB Docker version: 18.09 -Nvidia Driver version: 430.34 +NVIDIA Driver version: 430.34 Milvus version: 0.5.3 SDK interface: Python 3.6.8 -Pymilvus version: 0.2.5 +pymilvus version: 0.2.5 @@ -66,7 +66,7 @@ For details on this dataset, you can check : http://corpus-texmex.irisa.fr/ . > Note: In the query test of recall, we will test the following parameters with different values: > - > nq - grouped by: [1, 5, 10, 200, 400, 600, 800, 1000], + > nq - grouped by: [10, 200, 400, 600, 800, 1000], > > topk - grouped by: [1, 10, 100] @@ -93,7 +93,7 @@ Milvus configuration - gpu_cache_capacity: 6 - use_blas_threshold: 1100 -You can check the definition of Milvus configuration on https://milvus.io/docs/en/reference/milvus_config/. +The definitions of Milvus configuration are on https://milvus.io/docs/en/reference/milvus_config/. Test method diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md index bf43d2e0e8..a6e5e75ea4 100644 --- a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md @@ -2,7 +2,7 @@ ## 概述 -本文描述了ivfsq8索引在milvus单机部署方式下的测试报告。 +本文描述了ivfsq8索引在milvus单机部署方式下的测试结果。 @@ -28,13 +28,13 @@ GPU1: GeForce GTX 1080 Docker版本: 18.09 -Nvidia Driver版本: 430.34 +NVIDIA Driver版本: 430.34 Milvus版本: 0.5.3 SDK接口: Python 3.6.8 -Pymilvus版本: 0.2.5 +pymilvus版本: 0.2.5 @@ -66,7 +66,7 @@ Pymilvus版本: 0.2.5 > 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果： > - > 被查询向量的数量nq将按照 [1, 5, 10, 200, 400, 600, 800, 1000]的数量分组， + > 被查询向量的数量nq将按照 [10, 200, 400, 600, 800, 1000]的数量分组， > > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。 @@ -93,7 +93,7 @@ Milvus设置： - gpu_cache_capacity: 6 - use_blas_threshold: 1100 -你可以在 https://milvus.io/docs/en/reference/milvus_config/上查询Milvus设置的详细定义。 +Milvus设置的详细定义可以参考 https://milvus.io/docs/en/reference/milvus_config/ 。 测试方法 From 99deaf5c503b20b6af135549bc94a2b28c7d9da1 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Tue, 19 Nov 2019 17:37:13 +0800 Subject: [PATCH 32/56] modify shards for v0.5.3 --- shards/mishards/connections.py | 10 +++ shards/mishards/service_handler.py | 107 +++++++++++++++++++++-------- 2 files changed, 89 insertions(+), 28 deletions(-) diff --git a/shards/mishards/connections.py b/shards/mishards/connections.py index 618690a099..50e214ec9a 100644 --- a/shards/mishards/connections.py +++ b/shards/mishards/connections.py @@ -2,6 +2,7 @@ import logging import threading from functools import wraps from milvus import Milvus +from milvus.client.hooks import BaseaSearchHook from mishards import (settings, exceptions) from utils import singleton @@ -9,6 +10,12 @@ from utils import singleton logger = logging.getLogger(__name__) +class Searchook(BaseaSearchHook): + + def on_response(self, *args, **kwargs): + return True + + class Connection: def __init__(self, name, uri, max_retry=1, error_handlers=None, **kwargs): self.name = name @@ -18,6 +25,9 @@ class Connection: self.conn = Milvus() self.error_handlers = [] if not error_handlers else error_handlers self.on_retry_func = kwargs.get('on_retry_func', None) + + # define search hook + self.conn._set_hook(search_in_file=Searchook()) # self._connect() def __str__(self): diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 2f19152ae6..620f6213de 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -29,39 +29,88 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): self.router = router self.max_workers = max_workers + def _reduce(self, source_ids, ids, source_diss, diss, k, reverse): + if source_diss[k - 1] <= diss[0]: + return source_ids, source_diss + if diss[k - 1] <= source_diss[0]: + return ids, diss + + diss_t = enumerate(source_diss.extend(diss)) + diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] + diss_m_out = [id_ for _, id_ in diss_m_rst] + + id_t = source_ids.extend(ids) + id_m_out = [id_t[i] for i, _ in diss_m_rst] + + return id_m_out, diss_m_out + def _do_merge(self, files_n_topk_results, topk, reverse=False, **kwargs): status = status_pb2.Status(error_code=status_pb2.SUCCESS, reason="Success") if not files_n_topk_results: return status, [] - request_results = defaultdict(list) + # request_results = defaultdict(list) + # row_num = files_n_topk_results[0].row_num + merge_id_results = [] + merge_dis_results = [] calc_time = time.time() for files_collection in files_n_topk_results: if isinstance(files_collection, tuple): status, _ = files_collection return status, [] - for request_pos, each_request_results in enumerate( - files_collection.topk_query_result): - request_results[request_pos].extend( - each_request_results.query_result_arrays) - request_results[request_pos] = sorted( - request_results[request_pos], - key=lambda x: x.distance, - reverse=reverse)[:topk] + + row_num = files_collection.row_num + ids = files_collection.ids + diss = files_collection.distances # distance collections + batch_len = len(ids) // row_num + + for row_index in range(row_num): + id_batch = ids[row_index * batch_len: (row_index + 1) * batch_len] + dis_batch = diss[row_index * batch_len: (row_index + 1) * batch_len] + + if len(merge_id_results) < row_index: + raise ValueError("merge error") + elif len(merge_id_results) == row_index: + # TODO: may bug here + merge_id_results.append(id_batch) + merge_dis_results.append(dis_batch) + else: + merge_id_results[row_index].extend(ids[row_index * batch_len, (row_index + 1) * batch_len]) + merge_dis_results[row_index].extend(diss[row_index * batch_len, (row_index + 1) * batch_len]) + # _reduce(_ids, _diss, k, reverse) + merge_id_results[row_index], merge_dis_results[row_index] = \ + self._reduce(merge_id_results[row_index], id_batch, + merge_dis_results[row_index], dis_batch, + batch_len, + reverse) + + # for request_pos, each_request_results in enumerate( + # files_collection.topk_query_result): + # request_results[request_pos].extend( + # each_request_results.query_result_arrays) + # request_results[request_pos] = sorted( + # request_results[request_pos], + # key=lambda x: x.distance, + # reverse=reverse)[:topk] calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - results = sorted(request_results.items()) - topk_query_result = [] + # results = sorted(request_results.items()) + id_mrege_list = [] + dis_mrege_list = [] - for result in results: - query_result = TopKQueryResult(query_result_arrays=result[1]) - topk_query_result.append(query_result) + for id_results, dis_results in zip(merge_id_results, merge_dis_results): + id_mrege_list.extend(id_results) + dis_mrege_list.extend(dis_results) - return status, topk_query_result + # for result in results: + # query_result = TopKQueryResult(query_result_arrays=result[1]) + # topk_query_result.append(query_result) + + return status, id_mrege_list, dis_mrege_list def _do_query(self, context, @@ -109,8 +158,8 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): file_ids=query_params['file_ids'], query_records=vectors, top_k=topk, - nprobe=nprobe, - lazy_=True) + nprobe=nprobe + ) end = time.time() logger.info('search_vectors_in_files takes: {}'.format(end - start)) @@ -241,7 +290,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): logger.info('Search {}: topk={} nprobe={}'.format( table_name, topk, nprobe)) - metadata = {'resp_class': milvus_pb2.TopKQueryResultList} + metadata = {'resp_class': milvus_pb2.TopKQueryResult} if nprobe > self.MAX_NPROBE or nprobe <= 0: raise exceptions.InvalidArgumentError( @@ -275,22 +324,24 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): query_range_array.append( Range(query_range.start_value, query_range.end_value)) - status, results = self._do_query(context, - table_name, - table_meta, - query_record_array, - topk, - nprobe, - query_range_array, - metadata=metadata) + status, id_results, dis_results = self._do_query(context, + table_name, + table_meta, + query_record_array, + topk, + nprobe, + query_range_array, + metadata=metadata) now = time.time() logger.info('SearchVector takes: {}'.format(now - start)) - topk_result_list = milvus_pb2.TopKQueryResultList( + topk_result_list = milvus_pb2.TopKQueryResult( status=status_pb2.Status(error_code=status.error_code, reason=status.reason), - topk_query_result=results) + row_num=len(query_record_array), + ids=id_results, + distances=dis_results) return topk_result_list @mark_grpc_method From c451f8cbf2b66c45759f7ca7bba15212343e2a91 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Tue, 19 Nov 2019 19:40:17 +0800 Subject: [PATCH 33/56] clange format and add FallbackPass --- core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h | 1 - core/src/scheduler/SchedInst.h | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h index 4bb4092b75..e064b6f08c 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h @@ -47,7 +47,6 @@ class IVF : public VectorIndex, public FaissBaseIndex { void set_index_model(IndexModelPtr model) override; - void Add(const DatasetPtr& dataset, const Config& config) override; diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 48eaffc0a5..44fea1b6d3 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -104,14 +104,13 @@ class OptimizerInst { pass_list.push_back(std::make_shared()); #ifdef MILVUS_CPU_VERSION pass_list.push_back(std::make_shared()); -<<<<<<< HEAD #else server::Config& config = server::Config::GetInstance(); std::vector build_resources; config.GetGpuResourceConfigBuildIndexResources(build_resources); pass_list.push_back(std::make_shared(build_resources)); #endif -// pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); } } From fdb7decdd3debbeae3d74cd7fd5ec608b319ca19 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Tue, 19 Nov 2019 20:30:43 +0800 Subject: [PATCH 34/56] Remove DefaultLabel --- core/src/scheduler/JobMgr.cpp | 8 +-- core/src/scheduler/JobMgr.h | 5 +- core/src/scheduler/Scheduler.cpp | 4 -- core/src/scheduler/TaskCreator.cpp | 9 +-- core/src/scheduler/Utils.cpp | 1 - core/src/scheduler/action/Action.h | 4 -- .../scheduler/action/PushTaskToNeighbour.cpp | 32 --------- core/src/scheduler/tasklabel/DefaultLabel.h | 36 ---------- core/src/scheduler/tasklabel/TaskLabel.h | 1 - core/unittest/scheduler/CMakeLists.txt | 1 - core/unittest/scheduler/test_normal.cpp | 72 ------------------- core/unittest/scheduler/test_resource.cpp | 18 +++-- core/unittest/scheduler/test_resource_mgr.cpp | 4 +- core/unittest/scheduler/test_scheduler.cpp | 41 ----------- core/unittest/scheduler/test_tasktable.cpp | 9 +-- 15 files changed, 26 insertions(+), 219 deletions(-) delete mode 100644 core/src/scheduler/tasklabel/DefaultLabel.h delete mode 100644 core/unittest/scheduler/test_normal.cpp diff --git a/core/src/scheduler/JobMgr.cpp b/core/src/scheduler/JobMgr.cpp index 8dd095a3fa..76c07fe459 100644 --- a/core/src/scheduler/JobMgr.cpp +++ b/core/src/scheduler/JobMgr.cpp @@ -85,7 +85,7 @@ JobMgr::worker_function() { } for (auto& task : tasks) { - calculate_path(task); + calculate_path(res_mgr_, task); } // disk resources NEVER be empty. @@ -103,7 +103,7 @@ JobMgr::build_task(const JobPtr& job) { } void -JobMgr::calculate_path(const TaskPtr& task) { +JobMgr::calculate_path(const ResourceMgrPtr& res_mgr, const TaskPtr& task) { if (task->type_ != TaskType::SearchTask && task->type_ != TaskType::BuildIndexTask) { return; } @@ -114,9 +114,9 @@ JobMgr::calculate_path(const TaskPtr& task) { std::vector path; auto spec_label = std::static_pointer_cast(task->label()); - auto src = res_mgr_->GetDiskResources()[0]; + auto src = res_mgr->GetDiskResources()[0]; auto dest = spec_label->resource(); - ShortestPath(src.lock(), dest.lock(), res_mgr_, path); + ShortestPath(src.lock(), dest.lock(), res_mgr, path); task->path() = Path(path, path.size() - 1); } diff --git a/core/src/scheduler/JobMgr.h b/core/src/scheduler/JobMgr.h index fbd6c0ee45..af072614b5 100644 --- a/core/src/scheduler/JobMgr.h +++ b/core/src/scheduler/JobMgr.h @@ -59,8 +59,9 @@ class JobMgr : public interface::dumpable { static std::vector build_task(const JobPtr& job); - void - calculate_path(const TaskPtr& task); + public: + static void + calculate_path(const ResourceMgrPtr& res_mgr, const TaskPtr& task); private: bool running_ = false; diff --git a/core/src/scheduler/Scheduler.cpp b/core/src/scheduler/Scheduler.cpp index 8d2d4406f8..68d7457aa9 100644 --- a/core/src/scheduler/Scheduler.cpp +++ b/core/src/scheduler/Scheduler.cpp @@ -108,10 +108,6 @@ Scheduler::OnLoadCompleted(const EventPtr& event) { auto task_table_type = load_completed_event->task_table_item_->task->label()->Type(); switch (task_table_type) { - case TaskLabelType::DEFAULT: { - Action::DefaultLabelTaskScheduler(res_mgr_, resource, load_completed_event); - break; - } case TaskLabelType::SPECIFIED_RESOURCE: { Action::SpecifiedResourceLabelTaskScheduler(res_mgr_, resource, load_completed_event); break; diff --git a/core/src/scheduler/TaskCreator.cpp b/core/src/scheduler/TaskCreator.cpp index 3b7f6fd82e..30b76cc5bb 100644 --- a/core/src/scheduler/TaskCreator.cpp +++ b/core/src/scheduler/TaskCreator.cpp @@ -18,7 +18,6 @@ #include "scheduler/TaskCreator.h" #include "SchedInst.h" #include "tasklabel/BroadcastLabel.h" -#include "tasklabel/DefaultLabel.h" #include "tasklabel/SpecResLabel.h" namespace milvus { @@ -47,8 +46,7 @@ std::vector TaskCreator::Create(const SearchJobPtr& job) { std::vector tasks; for (auto& index_file : job->index_files()) { - auto label = std::make_shared(); - auto task = std::make_shared(index_file.second, label); + auto task = std::make_shared(index_file.second, nullptr); task->job_ = job; tasks.emplace_back(task); } @@ -70,11 +68,8 @@ TaskCreator::Create(const DeleteJobPtr& job) { std::vector TaskCreator::Create(const BuildIndexJobPtr& job) { std::vector tasks; - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); - for (auto& to_index_file : job->to_index_files()) { - auto label = std::make_shared(std::weak_ptr(res_ptr)); - auto task = std::make_shared(to_index_file.second, label); + auto task = std::make_shared(to_index_file.second, nullptr); task->job_ = job; tasks.emplace_back(task); } diff --git a/core/src/scheduler/Utils.cpp b/core/src/scheduler/Utils.cpp index 1be04ba80b..527b9adf03 100644 --- a/core/src/scheduler/Utils.cpp +++ b/core/src/scheduler/Utils.cpp @@ -16,7 +16,6 @@ // under the License. #include "scheduler/Utils.h" -#include "server/Config.h" #ifdef MILVUS_GPU_VERSION #include diff --git a/core/src/scheduler/action/Action.h b/core/src/scheduler/action/Action.h index f5f828cbf6..7391d287a8 100644 --- a/core/src/scheduler/action/Action.h +++ b/core/src/scheduler/action/Action.h @@ -36,10 +36,6 @@ class Action { static void PushTaskToResource(TaskTableItemPtr task_item, const ResourcePtr& dest); - static void - DefaultLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, - std::shared_ptr event); - static void SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, std::shared_ptr event); diff --git a/core/src/scheduler/action/PushTaskToNeighbour.cpp b/core/src/scheduler/action/PushTaskToNeighbour.cpp index 9aed678937..f49f1d871f 100644 --- a/core/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/core/src/scheduler/action/PushTaskToNeighbour.cpp @@ -101,38 +101,6 @@ Action::PushTaskToResource(TaskTableItemPtr task_item, const ResourcePtr& dest) dest->task_table().Put(task_item->task, task_item); } -void -Action::DefaultLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, - std::shared_ptr event) { - if (not resource->HasExecutor() && event->task_table_item_->Move()) { - auto task_item = event->task_table_item_; - auto task = event->task_table_item_->task; - auto search_task = std::static_pointer_cast(task); - bool moved = false; - - // to support test task, REFACTOR - if (resource->type() == ResourceType::CPU) { - if (auto index_engine = search_task->index_engine_) { - auto location = index_engine->GetLocation(); - - for (auto i = 0; i < res_mgr->GetNumGpuResource(); ++i) { - auto index = milvus::cache::GpuCacheMgr::GetInstance(i)->GetIndex(location); - if (index != nullptr) { - moved = true; - auto dest_resource = res_mgr->GetResource(ResourceType::GPU, i); - PushTaskToResource(event->task_table_item_, dest_resource); - break; - } - } - } - } - - if (not moved) { - PushTaskToNeighbourRandomly(task_item, resource); - } - } -} - void Action::SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, std::shared_ptr event) { diff --git a/core/src/scheduler/tasklabel/DefaultLabel.h b/core/src/scheduler/tasklabel/DefaultLabel.h deleted file mode 100644 index c215743575..0000000000 --- a/core/src/scheduler/tasklabel/DefaultLabel.h +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "TaskLabel.h" - -#include - -namespace milvus { -namespace scheduler { - -class DefaultLabel : public TaskLabel { - public: - DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) { - } -}; - -using DefaultLabelPtr = std::shared_ptr; - -} // namespace scheduler -} // namespace milvus diff --git a/core/src/scheduler/tasklabel/TaskLabel.h b/core/src/scheduler/tasklabel/TaskLabel.h index d35ce409ff..33e6eb6e57 100644 --- a/core/src/scheduler/tasklabel/TaskLabel.h +++ b/core/src/scheduler/tasklabel/TaskLabel.h @@ -23,7 +23,6 @@ namespace milvus { namespace scheduler { enum class TaskLabelType { - DEFAULT, // means can be executed in any resource SPECIFIED_RESOURCE, // means must executing in special resource BROADCAST, // means all enable-executor resource must execute task }; diff --git a/core/unittest/scheduler/CMakeLists.txt b/core/unittest/scheduler/CMakeLists.txt index 0148441c2d..878f0f23a9 100644 --- a/core/unittest/scheduler/CMakeLists.txt +++ b/core/unittest/scheduler/CMakeLists.txt @@ -21,7 +21,6 @@ set(test_files ${CMAKE_CURRENT_SOURCE_DIR}/test_algorithm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_node.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/test_normal.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_resource.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_resource_factory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_resource_mgr.cpp diff --git a/core/unittest/scheduler/test_normal.cpp b/core/unittest/scheduler/test_normal.cpp deleted file mode 100644 index 20cd39d0bf..0000000000 --- a/core/unittest/scheduler/test_normal.cpp +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include "scheduler/ResourceFactory.h" -#include "scheduler/ResourceMgr.h" -#include "scheduler/SchedInst.h" -#include "scheduler/Scheduler.h" -#include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" -#include "utils/Log.h" - -namespace { - -namespace ms = milvus::scheduler; - -} // namespace - -TEST(NormalTest, INST_TEST) { - // ResourceMgr only compose resources, provide unified event - auto res_mgr = ms::ResMgrInst::GetInstance(); - - res_mgr->Add(ms::ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(ms::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - - auto IO = ms::Connection("IO", 500.0); - res_mgr->Connect("disk", "cpu", IO); - - auto scheduler = ms::SchedInst::GetInstance(); - - res_mgr->Start(); - scheduler->Start(); - - const uint64_t NUM_TASK = 2; - std::vector> tasks; - ms::TableFileSchemaPtr dummy = nullptr; - - auto disks = res_mgr->GetDiskResources(); - ASSERT_FALSE(disks.empty()); - if (auto observe = disks[0].lock()) { - for (uint64_t i = 0; i < NUM_TASK; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); - task->label() = std::make_shared(); - tasks.push_back(task); - observe->task_table().Put(task); - } - } - - for (auto& task : tasks) { - task->Wait(); - ASSERT_EQ(task->load_count_, 1); - ASSERT_EQ(task->exec_count_, 1); - } - - scheduler->Stop(); - res_mgr->Stop(); -} diff --git a/core/unittest/scheduler/test_resource.cpp b/core/unittest/scheduler/test_resource.cpp index 6fe85ec558..101dcd3976 100644 --- a/core/unittest/scheduler/test_resource.cpp +++ b/core/unittest/scheduler/test_resource.cpp @@ -24,7 +24,7 @@ #include "scheduler/resource/TestResource.h" #include "scheduler/task/Task.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" +#include "scheduler/tasklabel/SpecResLabel.h" namespace milvus { namespace scheduler { @@ -182,8 +182,10 @@ TEST_F(ResourceAdvanceTest, DISK_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(disk_resource_); auto task = std::make_shared(dummy, label); + std::vector path{disk_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); disk_resource_->task_table().Put(task); } @@ -208,8 +210,10 @@ TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(cpu_resource_); auto task = std::make_shared(dummy, label); + std::vector path{cpu_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); cpu_resource_->task_table().Put(task); } @@ -234,8 +238,10 @@ TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(gpu_resource_); auto task = std::make_shared(dummy, label); + std::vector path{gpu_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); gpu_resource_->task_table().Put(task); } @@ -260,8 +266,10 @@ TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(test_resource_); auto task = std::make_shared(dummy, label); + std::vector path{test_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); test_resource_->task_table().Put(task); } diff --git a/core/unittest/scheduler/test_resource_mgr.cpp b/core/unittest/scheduler/test_resource_mgr.cpp index b9127060bd..d6495971c4 100644 --- a/core/unittest/scheduler/test_resource_mgr.cpp +++ b/core/unittest/scheduler/test_resource_mgr.cpp @@ -22,7 +22,6 @@ #include "scheduler/resource/GpuResource.h" #include "scheduler/resource/TestResource.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" namespace milvus { namespace scheduler { @@ -187,8 +186,7 @@ TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) { auto callback = [&](EventPtr event) { flag = true; }; mgr1_->RegisterSubscriber(callback); TableFileSchemaPtr dummy = nullptr; - auto label = std::make_shared(); - disk_res->task_table().Put(std::make_shared(dummy, label)); + disk_res->task_table().Put(std::make_shared(dummy, nullptr)); sleep(1); ASSERT_TRUE(flag); } diff --git a/core/unittest/scheduler/test_scheduler.cpp b/core/unittest/scheduler/test_scheduler.cpp index 22fb9be723..72538113c3 100644 --- a/core/unittest/scheduler/test_scheduler.cpp +++ b/core/unittest/scheduler/test_scheduler.cpp @@ -23,7 +23,6 @@ #include "scheduler/Scheduler.h" #include "scheduler/resource/Resource.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" #include "scheduler/tasklabel/SpecResLabel.h" #include "utils/Error.h" #include "wrapper/VecIndex.h" @@ -150,46 +149,6 @@ insert_dummy_index_into_gpu_cache(uint64_t device_id) { cache::GpuCacheMgr::GetInstance(device_id)->InsertItem("location", obj); } -TEST_F(SchedulerTest, ON_LOAD_COMPLETED) { - const uint64_t NUM = 10; - std::vector> tasks; - TableFileSchemaPtr dummy = std::make_shared(); - dummy->location_ = "location"; - - insert_dummy_index_into_gpu_cache(1); - - for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); - task->label() = std::make_shared(); - tasks.push_back(task); - cpu_resource_.lock()->task_table().Put(task); - } - - sleep(3); - ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().size(), NUM); -} - -TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) { - const uint64_t NUM = 10; - std::vector> tasks; - TableFileSchemaPtr dummy1 = std::make_shared(); - dummy1->location_ = "location"; - - tasks.clear(); - - for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy1, label); - task->label() = std::make_shared(); - tasks.push_back(task); - cpu_resource_.lock()->task_table().Put(task); - } - - sleep(3); - // ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM); -} - class SchedulerTest2 : public testing::Test { protected: void diff --git a/core/unittest/scheduler/test_tasktable.cpp b/core/unittest/scheduler/test_tasktable.cpp index 28a2e29c98..3fa1beabeb 100644 --- a/core/unittest/scheduler/test_tasktable.cpp +++ b/core/unittest/scheduler/test_tasktable.cpp @@ -18,7 +18,6 @@ #include #include "scheduler/TaskTable.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" /************ TaskTableBaseTest ************/ @@ -162,9 +161,8 @@ class TaskTableBaseTest : public ::testing::Test { SetUp() override { milvus::scheduler::TableFileSchemaPtr dummy = nullptr; invalid_task_ = nullptr; - auto label = std::make_shared(); - task1_ = std::make_shared(dummy, label); - task2_ = std::make_shared(dummy, label); + task1_ = std::make_shared(dummy, nullptr); + task2_ = std::make_shared(dummy, nullptr); } milvus::scheduler::TaskPtr invalid_task_; @@ -320,8 +318,7 @@ class TaskTableAdvanceTest : public ::testing::Test { SetUp() override { milvus::scheduler::TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < 8; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); + auto task = std::make_shared(dummy, nullptr); table1_.Put(task); } From 67605968b8977ddb9e29c7f0a2ea8d49a3e4f703 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Tue, 19 Nov 2019 20:36:08 +0800 Subject: [PATCH 35/56] finish results reduce in mishards --- shards/mishards/service_handler.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 620f6213de..640ae61ba8 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -34,13 +34,14 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): return source_ids, source_diss if diss[k - 1] <= source_diss[0]: return ids, diss - - diss_t = enumerate(source_diss.extend(diss)) + + source_diss.extend(diss) + diss_t = enumerate(source_diss) diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] diss_m_out = [id_ for _, id_ in diss_m_rst] - id_t = source_ids.extend(ids) - id_m_out = [id_t[i] for i, _ in diss_m_rst] + source_ids.extend(ids) + id_m_out = [source_ids[i] for i, _ in diss_m_rst] return id_m_out, diss_m_out @@ -50,8 +51,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): if not files_n_topk_results: return status, [] - # request_results = defaultdict(list) - # row_num = files_n_topk_results[0].row_num merge_id_results = [] merge_dis_results = [] @@ -64,6 +63,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): row_num = files_collection.row_num ids = files_collection.ids diss = files_collection.distances # distance collections + # TODO: batch_len is equal to topk batch_len = len(ids) // row_num for row_index in range(row_num): @@ -77,28 +77,16 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): merge_id_results.append(id_batch) merge_dis_results.append(dis_batch) else: - merge_id_results[row_index].extend(ids[row_index * batch_len, (row_index + 1) * batch_len]) - merge_dis_results[row_index].extend(diss[row_index * batch_len, (row_index + 1) * batch_len]) - # _reduce(_ids, _diss, k, reverse) merge_id_results[row_index], merge_dis_results[row_index] = \ self._reduce(merge_id_results[row_index], id_batch, merge_dis_results[row_index], dis_batch, batch_len, reverse) - # for request_pos, each_request_results in enumerate( - # files_collection.topk_query_result): - # request_results[request_pos].extend( - # each_request_results.query_result_arrays) - # request_results[request_pos] = sorted( - # request_results[request_pos], - # key=lambda x: x.distance, - # reverse=reverse)[:topk] calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - # results = sorted(request_results.items()) id_mrege_list = [] dis_mrege_list = [] @@ -106,10 +94,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): id_mrege_list.extend(id_results) dis_mrege_list.extend(dis_results) - # for result in results: - # query_result = TopKQueryResult(query_result_arrays=result[1]) - # topk_query_result.append(query_result) - return status, id_mrege_list, dis_mrege_list def _do_query(self, From a13677b65f252b9c4025c4c2e3a25dfb8e6e112c Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Tue, 19 Nov 2019 21:10:43 +0800 Subject: [PATCH 36/56] Refactor optimizer --- core/src/scheduler/SchedInst.h | 22 +++--- .../scheduler/optimizer/BuildIndexPass.cpp | 9 ++- core/src/scheduler/optimizer/BuildIndexPass.h | 2 +- .../src/scheduler/optimizer/FaissFlatPass.cpp | 69 +++++++++++++++++++ .../{LargeSQ8HPass.h => FaissFlatPass.h} | 6 +- .../{OnlyCPUPass.cpp => FaissIVFFlatPass.cpp} | 41 ++++++++--- .../{HybridPass.h => FaissIVFFlatPass.h} | 12 +++- ...LargeSQ8HPass.cpp => FaissIVFSQ8HPass.cpp} | 40 +++-------- .../scheduler/optimizer/FaissIVFSQ8HPass.h | 56 +++++++++++++++ .../scheduler/optimizer/FaissIVFSQ8Pass.cpp | 69 +++++++++++++++++++ .../{OnlyCPUPass.h => FaissIVFSQ8Pass.h} | 12 +++- core/src/scheduler/optimizer/HybridPass.cpp | 47 ------------- 12 files changed, 273 insertions(+), 112 deletions(-) create mode 100644 core/src/scheduler/optimizer/FaissFlatPass.cpp rename core/src/scheduler/optimizer/{LargeSQ8HPass.h => FaissFlatPass.h} (91%) rename core/src/scheduler/optimizer/{OnlyCPUPass.cpp => FaissIVFFlatPass.cpp} (56%) rename core/src/scheduler/optimizer/{HybridPass.h => FaissIVFFlatPass.h} (81%) rename core/src/scheduler/optimizer/{LargeSQ8HPass.cpp => FaissIVFSQ8HPass.cpp} (61%) create mode 100644 core/src/scheduler/optimizer/FaissIVFSQ8HPass.h create mode 100644 core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp rename core/src/scheduler/optimizer/{OnlyCPUPass.h => FaissIVFSQ8Pass.h} (81%) delete mode 100644 core/src/scheduler/optimizer/HybridPass.cpp diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 44fea1b6d3..912f5edf8d 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -24,9 +24,10 @@ #include "Utils.h" #include "optimizer/BuildIndexPass.h" #include "optimizer/FallbackPass.h" -#include "optimizer/HybridPass.h" -#include "optimizer/LargeSQ8HPass.h" -#include "optimizer/OnlyCPUPass.h" +#include "optimizer/FaissFlatPass.h" +#include "optimizer/FaissIVFFlatPass.h" +#include "optimizer/FaissIVFSQ8Pass.h" +#include "optimizer/FaissIVFSQ8HPass.h" #include "optimizer/Optimizer.h" #include "server/Config.h" @@ -100,15 +101,12 @@ class OptimizerInst { std::lock_guard lock(mutex_); if (instance == nullptr) { std::vector pass_list; - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); -#ifdef MILVUS_CPU_VERSION - pass_list.push_back(std::make_shared()); -#else - server::Config& config = server::Config::GetInstance(); - std::vector build_resources; - config.GetGpuResourceConfigBuildIndexResources(build_resources); - pass_list.push_back(std::make_shared(build_resources)); +#ifdef MILVUS_GPU_VERSION + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); #endif pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index 4e7d5222e2..cd1b7550d8 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -23,11 +23,14 @@ namespace milvus { namespace scheduler { -BuildIndexPass::BuildIndexPass(std::vector& build_gpu_ids) : build_gpu_ids_(build_gpu_ids) { -} - void BuildIndexPass::Init() { + server::Config& config = server::Config::GetInstance(); + std::vector build_resources; + Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); + if (!s.ok()) { + throw; + } } bool diff --git a/core/src/scheduler/optimizer/BuildIndexPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h index a3eb426527..4f7117fc4e 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.h +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -34,7 +34,7 @@ namespace scheduler { class BuildIndexPass : public Pass { public: - explicit BuildIndexPass(std::vector& build_gpu_id); + BuildIndexPass() = default; public: void diff --git a/core/src/scheduler/optimizer/FaissFlatPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp new file mode 100644 index 0000000000..61ca1b9ec9 --- /dev/null +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FaissFlatPass.h" +#include "cache/GpuCacheMgr.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" + +namespace milvus { +namespace scheduler { + +void +FaissFlatPass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } +} + +bool +FaissFlatPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { + return false; + } + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IDMAP) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/FaissFlatPass.h similarity index 91% rename from core/src/scheduler/optimizer/LargeSQ8HPass.h rename to core/src/scheduler/optimizer/FaissFlatPass.h index c69fb12c64..60adf62a07 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.h +++ b/core/src/scheduler/optimizer/FaissFlatPass.h @@ -33,9 +33,9 @@ namespace milvus { namespace scheduler { -class LargeSQ8HPass : public Pass { +class FaissFlatPass : public Pass { public: - LargeSQ8HPass() = default; + FaissFlatPass() = default; public: void @@ -50,7 +50,7 @@ class LargeSQ8HPass : public Pass { std::vector gpus; }; -using LargeSQ8HPassPtr = std::shared_ptr; +using FaissFlatPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp similarity index 56% rename from core/src/scheduler/optimizer/OnlyCPUPass.cpp rename to core/src/scheduler/optimizer/FaissIVFFlatPass.cpp index 8cd40092fb..1f1efb374b 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -15,31 +15,52 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/OnlyCPUPass.h" +#include "scheduler/optimizer/FaissIVFFlatPass.h" +#include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" #include "scheduler/task/SearchTask.h" #include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" namespace milvus { namespace scheduler { void -OnlyCPUPass::Init() { +FaissIVFFlatPass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } } bool -OnlyCPUPass::Run(const TaskPtr& task) { - if (task->Type() != TaskType::SearchTask) - return false; - auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8 && - search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { +FaissIVFFlatPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { return false; } - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); - auto label = std::make_shared(std::weak_ptr(res_ptr)); + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); task->label() = label; return true; } diff --git a/core/src/scheduler/optimizer/HybridPass.h b/core/src/scheduler/optimizer/FaissIVFFlatPass.h similarity index 81% rename from core/src/scheduler/optimizer/HybridPass.h rename to core/src/scheduler/optimizer/FaissIVFFlatPass.h index f84a0884f2..9264db32a8 100644 --- a/core/src/scheduler/optimizer/HybridPass.h +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -32,9 +33,9 @@ namespace milvus { namespace scheduler { -class HybridPass : public Pass { +class FaissIVFFlatPass : public Pass { public: - HybridPass() = default; + FaissIVFFlatPass() = default; public: void @@ -42,9 +43,14 @@ class HybridPass : public Pass { bool Run(const TaskPtr& task) override; + + private: + int32_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; }; -using HybridPassPtr = std::shared_ptr; +using FaissIVFFlatPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp similarity index 61% rename from core/src/scheduler/optimizer/LargeSQ8HPass.cpp rename to core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index dd58feb512..b63a0f7490 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/LargeSQ8HPass.h" +#include "scheduler/optimizer/FaissIVFSQ8HPass.h" #include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" @@ -28,7 +28,7 @@ namespace milvus { namespace scheduler { void -LargeSQ8HPass::Init() { +FaissIVFSQ8HPass::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { @@ -38,7 +38,7 @@ LargeSQ8HPass::Init() { } bool -LargeSQ8HPass::Run(const TaskPtr& task) { +FaissIVFSQ8HPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::SearchTask) { return false; } @@ -49,36 +49,16 @@ LargeSQ8HPass::Run(const TaskPtr& task) { } auto search_job = std::static_pointer_cast(search_task->job_.lock()); - - // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu - + ResourcePtr res_ptr; if (search_job->nq() < threshold_) { - return false; + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } - - // std::vector all_free_mem; - // for (auto& gpu : gpus) { - // auto cache = cache::GpuCacheMgr::GetInstance(gpu); - // auto free_mem = cache->CacheCapacity() - cache->CacheUsage(); - // all_free_mem.push_back(free_mem); - // } - // - // auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end()); - // auto best_index = std::distance(all_free_mem.begin(), max_e); - // auto best_device_id = gpus[best_index]; - auto best_device_id = count_ % gpus.size(); - count_++; - - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); - if (not res_ptr) { - SERVER_LOG_ERROR << "GpuResource " << best_device_id << " invalid."; - // TODO: throw critical error and exit - return false; - } - - auto label = std::make_shared(std::weak_ptr(res_ptr)); + auto label = std::make_shared(res_ptr); task->label() = label; - return true; } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h new file mode 100644 index 0000000000..fa830ff08a --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class FaissIVFSQ8HPass : public Pass { + public: + FaissIVFSQ8HPass() = default; + + public: + void + Init() override; + + bool + Run(const TaskPtr& task) override; + + private: + int32_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; +}; + +using FaissIVFSQ8HPassPtr = std::shared_ptr; + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp new file mode 100644 index 0000000000..30dd306b3b --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FaissIVFSQ8Pass.h" +#include "cache/GpuCacheMgr.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" + +namespace milvus { +namespace scheduler { + +void +FaissIVFSQ8Pass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } +} + +bool +FaissIVFSQ8Pass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { + return false; + } + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h similarity index 81% rename from core/src/scheduler/optimizer/OnlyCPUPass.h rename to core/src/scheduler/optimizer/FaissIVFSQ8Pass.h index 253775f77e..c5853283f7 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -32,9 +33,9 @@ namespace milvus { namespace scheduler { -class OnlyCPUPass : public Pass { +class FaissIVFSQ8Pass : public Pass { public: - OnlyCPUPass() = default; + FaissIVFSQ8Pass() = default; public: void @@ -42,9 +43,14 @@ class OnlyCPUPass : public Pass { bool Run(const TaskPtr& task) override; + + private: + int32_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; }; -using OnlyCPUPassPtr = std::shared_ptr; +using FaissIVFSQ8PassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/HybridPass.cpp b/core/src/scheduler/optimizer/HybridPass.cpp deleted file mode 100644 index 8bf0d8990d..0000000000 --- a/core/src/scheduler/optimizer/HybridPass.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "scheduler/optimizer/HybridPass.h" -#include "scheduler/SchedInst.h" -#include "scheduler/task/SearchTask.h" -#include "scheduler/tasklabel/SpecResLabel.h" - -namespace milvus { -namespace scheduler { - -void -HybridPass::Init() { -} - -bool -HybridPass::Run(const TaskPtr& task) { - // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu - if (task->Type() != TaskType::SearchTask) - return false; - auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ == (int)engine::EngineType::FAISS_IVFSQ8H) { - // TODO: remove "cpu" hardcode - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); - auto label = std::make_shared(std::weak_ptr(res_ptr)); - task->label() = label; - return true; - } - return false; -} - -} // namespace scheduler -} // namespace milvus From a176eb94a21b2cf015745167df0903f490709219 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Tue, 19 Nov 2019 21:12:12 +0800 Subject: [PATCH 37/56] clange format --- core/src/scheduler/SchedInst.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 912f5edf8d..dc2d5ade35 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -23,11 +23,11 @@ #include "Scheduler.h" #include "Utils.h" #include "optimizer/BuildIndexPass.h" -#include "optimizer/FallbackPass.h" #include "optimizer/FaissFlatPass.h" #include "optimizer/FaissIVFFlatPass.h" -#include "optimizer/FaissIVFSQ8Pass.h" #include "optimizer/FaissIVFSQ8HPass.h" +#include "optimizer/FaissIVFSQ8Pass.h" +#include "optimizer/FallbackPass.h" #include "optimizer/Optimizer.h" #include "server/Config.h" From 37df2b6fc42e1647f8dd760eacdfeb08e60210a4 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 10:22:42 +0800 Subject: [PATCH 38/56] #346 rename gpu_resource_config.enable_gpu to gpu_resource_config.enable --- core/conf/server_cpu_config.template | 2 +- core/conf/server_gpu_config.template | 2 +- core/src/server/Config.cpp | 57 ++++++++++++++-------------- core/src/server/Config.h | 10 ++--- core/unittest/server/test_config.cpp | 6 +-- 5 files changed, 38 insertions(+), 39 deletions(-) diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 1a384f19a1..41889f5cef 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -36,7 +36,7 @@ engine_config: gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only gpu_resource_config: - enable_gpu: false # whether to enable GPU resources + enable: false # whether to enable GPU resources cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer search_resources: # define the GPU devices used for search computation, must be in format gpux - gpu0 diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 9790c93491..531c633da7 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -36,7 +36,7 @@ engine_config: gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only gpu_resource_config: - enable_gpu: true # whether to enable GPU resources + enable: true # whether to enable GPU resources cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer search_resources: # define the GPU devices used for search computation, must be in format gpux - gpu0 diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index e3a48b6ae4..2ff91bfe3a 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -190,8 +190,8 @@ Config::ValidateConfig() { /* gpu resource config */ #ifdef MILVUS_GPU_VERSION - bool resource_enable_gpu; - s = GetGpuResourceConfigEnableGpu(resource_enable_gpu); + bool gpu_resource_enable; + s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } @@ -330,7 +330,7 @@ Config::ResetDefaultConfig() { /* gpu resource config */ #ifdef MILVUS_GPU_VERSION - s = SetGpuResourceConfigEnableGpu(CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT); + s = SetGpuResourceConfigEnable(CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); if (!s.ok()) { return s; } @@ -641,10 +641,10 @@ Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { } Status -Config::CheckGpuResourceConfigEnableGpu(const std::string& value) { +Config::CheckGpuResourceConfigEnable(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { - std::string msg = "Invalid gpu resource config: " + value + - ". Possible reason: gpu_resource_config.enable_gpu is not a boolean."; + std::string msg = + "Invalid gpu resource config: " + value + ". Possible reason: gpu_resource_config.enable is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -992,10 +992,9 @@ Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { } Status -Config::GetGpuResourceConfigEnableGpu(bool& value) { - std::string str = - GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE_GPU, CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT); - Status s = CheckGpuResourceConfigEnableGpu(str); +Config::GetGpuResourceConfigEnable(bool& value) { + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); + Status s = CheckGpuResourceConfigEnable(str); if (!s.ok()) { return s; } @@ -1006,13 +1005,13 @@ Config::GetGpuResourceConfigEnableGpu(bool& value) { Status Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, @@ -1027,13 +1026,13 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { Status Config::GetGpuResourceConfigCacheThreshold(float& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, @@ -1048,13 +1047,13 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) { Status Config::GetGpuResourceConfigSearchResources(std::vector& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, @@ -1073,13 +1072,13 @@ Config::GetGpuResourceConfigSearchResources(std::vector& value) { Status Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = @@ -1295,12 +1294,12 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { /* gpu resource config */ Status -Config::SetGpuResourceConfigEnableGpu(const std::string& value) { - Status s = CheckGpuResourceConfigEnableGpu(value); +Config::SetGpuResourceConfigEnable(const std::string& value) { + Status s = CheckGpuResourceConfigEnable(value); if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE_GPU, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, value); return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 0442ae0626..7c141be57e 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -85,9 +85,9 @@ static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; /* gpu resource config */ static const char* CONFIG_GPU_RESOURCE = "gpu_resource_config"; -static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU = "enable_gpu"; +static const char* CONFIG_GPU_RESOURCE_ENABLE = "enable"; #ifdef MILVUS_GPU_VERSION -static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "true"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; #else static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "false"; #endif @@ -175,7 +175,7 @@ class Config { /* gpu resource config */ Status - CheckGpuResourceConfigEnableGpu(const std::string& value); + CheckGpuResourceConfigEnable(const std::string& value); Status CheckGpuResourceConfigCacheCapacity(const std::string& value); Status @@ -244,7 +244,7 @@ class Config { /* gpu resource config */ Status - GetGpuResourceConfigEnableGpu(bool& value); + GetGpuResourceConfigEnable(bool& value); Status GetGpuResourceConfigCacheCapacity(int64_t& value); Status @@ -305,7 +305,7 @@ class Config { /* gpu resource config */ Status - SetGpuResourceConfigEnableGpu(const std::string& value); + SetGpuResourceConfigEnable(const std::string& value); Status SetGpuResourceConfigCacheCapacity(const std::string& value); Status diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 6ba7f19160..e0fde62916 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -246,9 +246,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { /* gpu resource config */ bool resource_enable_gpu = true; - s = config.SetGpuResourceConfigEnableGpu(std::to_string(resource_enable_gpu)); + s = config.SetGpuResourceConfigEnable(std::to_string(resource_enable_gpu)); ASSERT_TRUE(s.ok()); - s = config.GetGpuResourceConfigEnableGpu(bool_val); + s = config.GetGpuResourceConfigEnable(bool_val); ASSERT_TRUE(s.ok()); ASSERT_TRUE(bool_val == resource_enable_gpu); @@ -394,7 +394,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(s.ok()); /* gpu resource config */ - s = config.SetGpuResourceConfigEnableGpu("ok"); + s = config.SetGpuResourceConfigEnable("ok"); ASSERT_FALSE(s.ok()); #ifdef MILVUS_GPU_VERSION From f9ece14f0aa57b9465e5886f84f631b2762b18b2 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Fri, 15 Nov 2019 17:00:53 +0800 Subject: [PATCH 39/56] ignore proto python --- core/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/.gitignore b/core/.gitignore index 74e41dba6b..8db8df41db 100644 --- a/core/.gitignore +++ b/core/.gitignore @@ -9,3 +9,5 @@ output.info output_new.info server.info *.pyc +src/grpc/python_gen.h +src/grpc/python/ From 660953afa2d55ec7ba3d31f02a73e7d750803d95 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Sat, 16 Nov 2019 09:34:17 +0800 Subject: [PATCH 40/56] modify sdk --- .../sdk/examples/partition/src/ClientTest.cpp | 65 ++++++++++--------- core/src/sdk/examples/simple/main.cpp | 2 +- .../sdk/examples/simple/src/ClientTest.cpp | 3 +- core/src/sdk/grpc/ClientProxy.cpp | 30 ++++++--- core/src/sdk/grpc/ClientProxy.h | 2 +- 5 files changed, 58 insertions(+), 44 deletions(-) diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index 775e1f6d60..b67b1fe907 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -31,45 +31,45 @@ namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); + const char *TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; -constexpr int64_t BATCH_ROW_COUNT = 10000; -constexpr int64_t NQ = 5; -constexpr int64_t TOP_K = 10; -constexpr int64_t NPROBE = 32; -constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different -constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; -constexpr int32_t N_LIST = 15000; -constexpr int32_t PARTITION_COUNT = 5; -constexpr int32_t TARGET_PARTITION = 3; + constexpr int64_t TABLE_DIMENSION = 512; + constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; + constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; + constexpr int64_t BATCH_ROW_COUNT = 10000; + constexpr int64_t NQ = 5; + constexpr int64_t TOP_K = 10; + constexpr int64_t NPROBE = 32; + constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different + constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; + constexpr int32_t N_LIST = 15000; + constexpr int32_t PARTITION_COUNT = 5; + constexpr int32_t TARGET_PARTITION = 3; -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; -} + milvus::TableSchema + BuildTableSchema() { + milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; + return tb_schema; + } -milvus::PartitionParam -BuildPartitionParam(int32_t index) { - std::string tag = std::to_string(index); - std::string partition_name = std::string(TABLE_NAME) + "_" + tag; - milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; - return partition_param; -} + milvus::PartitionParam + BuildPartitionParam(int32_t index) { + std::string tag = std::to_string(index); + std::string partition_name = std::string(TABLE_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; + return partition_param; + } -milvus::IndexParam -BuildIndexParam() { - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; - return index_param; -} + milvus::IndexParam + BuildIndexParam() { + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + return index_param; + } } // namespace void -ClientTest::Test(const std::string& address, const std::string& port) { +ClientTest::Test(const std::string &address, const std::string &port) { std::shared_ptr conn = milvus::Connection::Create(); milvus::Status stat; @@ -78,7 +78,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - +#ifdef yhz { // create table milvus::TableSchema tb_schema = BuildTableSchema(); stat = conn->CreateTable(tb_schema); @@ -202,4 +202,5 @@ ClientTest::Test(const std::string& address, const std::string& port) { } milvus::Connection::Destroy(conn); +#endif } diff --git a/core/src/sdk/examples/simple/main.cpp b/core/src/sdk/examples/simple/main.cpp index c08741606c..d9b6194329 100644 --- a/core/src/sdk/examples/simple/main.cpp +++ b/core/src/sdk/examples/simple/main.cpp @@ -36,7 +36,7 @@ main(int argc, char* argv[]) { {nullptr, 0, nullptr, 0}}; int option_index = 0; - std::string address = "127.0.0.1", port = "19530"; + std::string address = "192.168.1.89", port = "19530"; app_name = argv[0]; int value; diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp index dfa5e2219e..da43f2f2aa 100644 --- a/core/src/sdk/examples/simple/src/ClientTest.cpp +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -68,7 +68,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - +#ifdef yhz { // server version std::string version = conn->ServerVersion(); std::cout << "Server version: " << version << std::endl; @@ -206,4 +206,5 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; } +#endif } diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 4a9c319b4d..1805ae81fe 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -43,16 +43,28 @@ Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; - channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - if (channel_ != nullptr) { - connected_ = true; - client_ptr_ = std::make_shared(channel_); - return Status::OK(); - } +// channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - std::string reason = "connect failed!"; - connected_ = false; - return Status(StatusCode::NotConnected, reason); +// channel_ = std::make_shared(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); +// channel_ = std::shared_ptr(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); + auto uri_str = uri.c_str(); + grpc_channel * channel = grpc_insecure_channel_create(uri_str, nullptr, nullptr); +// grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr); + auto state = grpc_channel_check_connectivity_state(channel, true); + if (state == GRPC_CHANNEL_READY) { + std::cout << "Connect " << uri << " successfully"; + } else { + std::cout << "Connect " << uri << " failed."; + } +// if (channel_ != nullptr) { +// connected_ = true; +// client_ptr_ = std::make_shared(channel_); +// return Status::OK(); +// } + +// std::string reason = "connect failed!"; +// connected_ = false; +// return Status(StatusCode::NotConnected, reason); } Status diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index e332266acf..572b782769 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -105,7 +105,7 @@ class ClientProxy : public Connection { DropPartition(const PartitionParam& partition_param) override; private: - std::shared_ptr<::grpc::Channel> channel_; + std::shared_ptr channel_; private: std::shared_ptr client_ptr_; From 83d9bf69662f4bbbbe51959be08f8858b1f3485d Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Tue, 19 Nov 2019 17:37:13 +0800 Subject: [PATCH 41/56] modify shards for v0.5.3 --- shards/mishards/connections.py | 10 +++ shards/mishards/service_handler.py | 107 +++++++++++++++++++++-------- 2 files changed, 89 insertions(+), 28 deletions(-) diff --git a/shards/mishards/connections.py b/shards/mishards/connections.py index 618690a099..50e214ec9a 100644 --- a/shards/mishards/connections.py +++ b/shards/mishards/connections.py @@ -2,6 +2,7 @@ import logging import threading from functools import wraps from milvus import Milvus +from milvus.client.hooks import BaseaSearchHook from mishards import (settings, exceptions) from utils import singleton @@ -9,6 +10,12 @@ from utils import singleton logger = logging.getLogger(__name__) +class Searchook(BaseaSearchHook): + + def on_response(self, *args, **kwargs): + return True + + class Connection: def __init__(self, name, uri, max_retry=1, error_handlers=None, **kwargs): self.name = name @@ -18,6 +25,9 @@ class Connection: self.conn = Milvus() self.error_handlers = [] if not error_handlers else error_handlers self.on_retry_func = kwargs.get('on_retry_func', None) + + # define search hook + self.conn._set_hook(search_in_file=Searchook()) # self._connect() def __str__(self): diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 2f19152ae6..620f6213de 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -29,39 +29,88 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): self.router = router self.max_workers = max_workers + def _reduce(self, source_ids, ids, source_diss, diss, k, reverse): + if source_diss[k - 1] <= diss[0]: + return source_ids, source_diss + if diss[k - 1] <= source_diss[0]: + return ids, diss + + diss_t = enumerate(source_diss.extend(diss)) + diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] + diss_m_out = [id_ for _, id_ in diss_m_rst] + + id_t = source_ids.extend(ids) + id_m_out = [id_t[i] for i, _ in diss_m_rst] + + return id_m_out, diss_m_out + def _do_merge(self, files_n_topk_results, topk, reverse=False, **kwargs): status = status_pb2.Status(error_code=status_pb2.SUCCESS, reason="Success") if not files_n_topk_results: return status, [] - request_results = defaultdict(list) + # request_results = defaultdict(list) + # row_num = files_n_topk_results[0].row_num + merge_id_results = [] + merge_dis_results = [] calc_time = time.time() for files_collection in files_n_topk_results: if isinstance(files_collection, tuple): status, _ = files_collection return status, [] - for request_pos, each_request_results in enumerate( - files_collection.topk_query_result): - request_results[request_pos].extend( - each_request_results.query_result_arrays) - request_results[request_pos] = sorted( - request_results[request_pos], - key=lambda x: x.distance, - reverse=reverse)[:topk] + + row_num = files_collection.row_num + ids = files_collection.ids + diss = files_collection.distances # distance collections + batch_len = len(ids) // row_num + + for row_index in range(row_num): + id_batch = ids[row_index * batch_len: (row_index + 1) * batch_len] + dis_batch = diss[row_index * batch_len: (row_index + 1) * batch_len] + + if len(merge_id_results) < row_index: + raise ValueError("merge error") + elif len(merge_id_results) == row_index: + # TODO: may bug here + merge_id_results.append(id_batch) + merge_dis_results.append(dis_batch) + else: + merge_id_results[row_index].extend(ids[row_index * batch_len, (row_index + 1) * batch_len]) + merge_dis_results[row_index].extend(diss[row_index * batch_len, (row_index + 1) * batch_len]) + # _reduce(_ids, _diss, k, reverse) + merge_id_results[row_index], merge_dis_results[row_index] = \ + self._reduce(merge_id_results[row_index], id_batch, + merge_dis_results[row_index], dis_batch, + batch_len, + reverse) + + # for request_pos, each_request_results in enumerate( + # files_collection.topk_query_result): + # request_results[request_pos].extend( + # each_request_results.query_result_arrays) + # request_results[request_pos] = sorted( + # request_results[request_pos], + # key=lambda x: x.distance, + # reverse=reverse)[:topk] calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - results = sorted(request_results.items()) - topk_query_result = [] + # results = sorted(request_results.items()) + id_mrege_list = [] + dis_mrege_list = [] - for result in results: - query_result = TopKQueryResult(query_result_arrays=result[1]) - topk_query_result.append(query_result) + for id_results, dis_results in zip(merge_id_results, merge_dis_results): + id_mrege_list.extend(id_results) + dis_mrege_list.extend(dis_results) - return status, topk_query_result + # for result in results: + # query_result = TopKQueryResult(query_result_arrays=result[1]) + # topk_query_result.append(query_result) + + return status, id_mrege_list, dis_mrege_list def _do_query(self, context, @@ -109,8 +158,8 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): file_ids=query_params['file_ids'], query_records=vectors, top_k=topk, - nprobe=nprobe, - lazy_=True) + nprobe=nprobe + ) end = time.time() logger.info('search_vectors_in_files takes: {}'.format(end - start)) @@ -241,7 +290,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): logger.info('Search {}: topk={} nprobe={}'.format( table_name, topk, nprobe)) - metadata = {'resp_class': milvus_pb2.TopKQueryResultList} + metadata = {'resp_class': milvus_pb2.TopKQueryResult} if nprobe > self.MAX_NPROBE or nprobe <= 0: raise exceptions.InvalidArgumentError( @@ -275,22 +324,24 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): query_range_array.append( Range(query_range.start_value, query_range.end_value)) - status, results = self._do_query(context, - table_name, - table_meta, - query_record_array, - topk, - nprobe, - query_range_array, - metadata=metadata) + status, id_results, dis_results = self._do_query(context, + table_name, + table_meta, + query_record_array, + topk, + nprobe, + query_range_array, + metadata=metadata) now = time.time() logger.info('SearchVector takes: {}'.format(now - start)) - topk_result_list = milvus_pb2.TopKQueryResultList( + topk_result_list = milvus_pb2.TopKQueryResult( status=status_pb2.Status(error_code=status.error_code, reason=status.reason), - topk_query_result=results) + row_num=len(query_record_array), + ids=id_results, + distances=dis_results) return topk_result_list @mark_grpc_method From 2f8be3d0586625d65f08622933d85fa8eabd6553 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Tue, 19 Nov 2019 20:36:08 +0800 Subject: [PATCH 42/56] finish results reduce in mishards --- shards/mishards/service_handler.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 620f6213de..640ae61ba8 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -34,13 +34,14 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): return source_ids, source_diss if diss[k - 1] <= source_diss[0]: return ids, diss - - diss_t = enumerate(source_diss.extend(diss)) + + source_diss.extend(diss) + diss_t = enumerate(source_diss) diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] diss_m_out = [id_ for _, id_ in diss_m_rst] - id_t = source_ids.extend(ids) - id_m_out = [id_t[i] for i, _ in diss_m_rst] + source_ids.extend(ids) + id_m_out = [source_ids[i] for i, _ in diss_m_rst] return id_m_out, diss_m_out @@ -50,8 +51,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): if not files_n_topk_results: return status, [] - # request_results = defaultdict(list) - # row_num = files_n_topk_results[0].row_num merge_id_results = [] merge_dis_results = [] @@ -64,6 +63,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): row_num = files_collection.row_num ids = files_collection.ids diss = files_collection.distances # distance collections + # TODO: batch_len is equal to topk batch_len = len(ids) // row_num for row_index in range(row_num): @@ -77,28 +77,16 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): merge_id_results.append(id_batch) merge_dis_results.append(dis_batch) else: - merge_id_results[row_index].extend(ids[row_index * batch_len, (row_index + 1) * batch_len]) - merge_dis_results[row_index].extend(diss[row_index * batch_len, (row_index + 1) * batch_len]) - # _reduce(_ids, _diss, k, reverse) merge_id_results[row_index], merge_dis_results[row_index] = \ self._reduce(merge_id_results[row_index], id_batch, merge_dis_results[row_index], dis_batch, batch_len, reverse) - # for request_pos, each_request_results in enumerate( - # files_collection.topk_query_result): - # request_results[request_pos].extend( - # each_request_results.query_result_arrays) - # request_results[request_pos] = sorted( - # request_results[request_pos], - # key=lambda x: x.distance, - # reverse=reverse)[:topk] calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - # results = sorted(request_results.items()) id_mrege_list = [] dis_mrege_list = [] @@ -106,10 +94,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): id_mrege_list.extend(id_results) dis_mrege_list.extend(dis_results) - # for result in results: - # query_result = TopKQueryResult(query_result_arrays=result[1]) - # topk_query_result.append(query_result) - return status, id_mrege_list, dis_mrege_list def _do_query(self, From b552b77a7eb6aaec2afc4ce67f452a00adea982d Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 10:53:28 +0800 Subject: [PATCH 43/56] [skip ci] add comments in shards --- shards/mishards/service_handler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 640ae61ba8..fc0ee0fa2b 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -63,7 +63,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): row_num = files_collection.row_num ids = files_collection.ids diss = files_collection.distances # distance collections - # TODO: batch_len is equal to topk + # TODO: batch_len is equal to topk, may need to compare with topk batch_len = len(ids) // row_num for row_index in range(row_num): @@ -83,7 +83,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): batch_len, reverse) - calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) From 631cf7143976f6e08bc4afb7064b46b773af8010 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 10:57:01 +0800 Subject: [PATCH 44/56] #346 fix CPU version build error --- core/src/server/Config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 7c141be57e..a6e78671b3 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -89,7 +89,7 @@ static const char* CONFIG_GPU_RESOURCE_ENABLE = "enable"; #ifdef MILVUS_GPU_VERSION static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; #else -static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "false"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "false"; #endif static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY = "cache_capacity"; static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "4"; From a5a41212f8489e3e69b4249dfcc1c1b42a13d513 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 11:06:24 +0800 Subject: [PATCH 45/56] remove test change' --- .../sdk/examples/partition/src/ClientTest.cpp | 3 +- core/src/sdk/examples/simple/main.cpp | 2 +- .../sdk/examples/simple/src/ClientTest.cpp | 3 +- core/src/sdk/grpc/ClientProxy.cpp | 28 ++++++------------- core/src/sdk/grpc/ClientProxy.h | 2 +- 5 files changed, 12 insertions(+), 26 deletions(-) diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index b67b1fe907..79ed7646b9 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -78,7 +78,7 @@ ClientTest::Test(const std::string &address, const std::string &port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } -#ifdef yhz + { // create table milvus::TableSchema tb_schema = BuildTableSchema(); stat = conn->CreateTable(tb_schema); @@ -202,5 +202,4 @@ ClientTest::Test(const std::string &address, const std::string &port) { } milvus::Connection::Destroy(conn); -#endif } diff --git a/core/src/sdk/examples/simple/main.cpp b/core/src/sdk/examples/simple/main.cpp index d9b6194329..c08741606c 100644 --- a/core/src/sdk/examples/simple/main.cpp +++ b/core/src/sdk/examples/simple/main.cpp @@ -36,7 +36,7 @@ main(int argc, char* argv[]) { {nullptr, 0, nullptr, 0}}; int option_index = 0; - std::string address = "192.168.1.89", port = "19530"; + std::string address = "127.0.0.1", port = "19530"; app_name = argv[0]; int value; diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp index da43f2f2aa..dfa5e2219e 100644 --- a/core/src/sdk/examples/simple/src/ClientTest.cpp +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -68,7 +68,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } -#ifdef yhz + { // server version std::string version = conn->ServerVersion(); std::cout << "Server version: " << version << std::endl; @@ -206,5 +206,4 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; } -#endif } diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 1805ae81fe..4a9c319b4d 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -43,28 +43,16 @@ Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; -// channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - -// channel_ = std::make_shared(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); -// channel_ = std::shared_ptr(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); - auto uri_str = uri.c_str(); - grpc_channel * channel = grpc_insecure_channel_create(uri_str, nullptr, nullptr); -// grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr); - auto state = grpc_channel_check_connectivity_state(channel, true); - if (state == GRPC_CHANNEL_READY) { - std::cout << "Connect " << uri << " successfully"; - } else { - std::cout << "Connect " << uri << " failed."; + channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); + if (channel_ != nullptr) { + connected_ = true; + client_ptr_ = std::make_shared(channel_); + return Status::OK(); } -// if (channel_ != nullptr) { -// connected_ = true; -// client_ptr_ = std::make_shared(channel_); -// return Status::OK(); -// } -// std::string reason = "connect failed!"; -// connected_ = false; -// return Status(StatusCode::NotConnected, reason); + std::string reason = "connect failed!"; + connected_ = false; + return Status(StatusCode::NotConnected, reason); } Status diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index 572b782769..e332266acf 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -105,7 +105,7 @@ class ClientProxy : public Connection { DropPartition(const PartitionParam& partition_param) override; private: - std::shared_ptr channel_; + std::shared_ptr<::grpc::Channel> channel_; private: std::shared_ptr client_ptr_; From ed5219b72804d86fc910495418411aa981348a49 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 11:23:48 +0800 Subject: [PATCH 46/56] [skip ci] foramt pass --- .../sdk/examples/partition/src/ClientTest.cpp | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index 79ed7646b9..775e1f6d60 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -31,45 +31,45 @@ namespace { - const char *TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); +const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); - constexpr int64_t TABLE_DIMENSION = 512; - constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; - constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; - constexpr int64_t BATCH_ROW_COUNT = 10000; - constexpr int64_t NQ = 5; - constexpr int64_t TOP_K = 10; - constexpr int64_t NPROBE = 32; - constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different - constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; - constexpr int32_t N_LIST = 15000; - constexpr int32_t PARTITION_COUNT = 5; - constexpr int32_t TARGET_PARTITION = 3; +constexpr int64_t TABLE_DIMENSION = 512; +constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; +constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; +constexpr int64_t BATCH_ROW_COUNT = 10000; +constexpr int64_t NQ = 5; +constexpr int64_t TOP_K = 10; +constexpr int64_t NPROBE = 32; +constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different +constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; +constexpr int32_t N_LIST = 15000; +constexpr int32_t PARTITION_COUNT = 5; +constexpr int32_t TARGET_PARTITION = 3; - milvus::TableSchema - BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; - } +milvus::TableSchema +BuildTableSchema() { + milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; + return tb_schema; +} - milvus::PartitionParam - BuildPartitionParam(int32_t index) { - std::string tag = std::to_string(index); - std::string partition_name = std::string(TABLE_NAME) + "_" + tag; - milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; - return partition_param; - } +milvus::PartitionParam +BuildPartitionParam(int32_t index) { + std::string tag = std::to_string(index); + std::string partition_name = std::string(TABLE_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; + return partition_param; +} - milvus::IndexParam - BuildIndexParam() { - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; - return index_param; - } +milvus::IndexParam +BuildIndexParam() { + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + return index_param; +} } // namespace void -ClientTest::Test(const std::string &address, const std::string &port) { +ClientTest::Test(const std::string& address, const std::string& port) { std::shared_ptr conn = milvus::Connection::Create(); milvus::Status stat; From 359197eaafad45e8b0d32df1cc1726b7ac2f8bd3 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 11:35:36 +0800 Subject: [PATCH 47/56] update requirements.txt --- shards/requirements.txt | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/shards/requirements.txt b/shards/requirements.txt index 14bdde2a06..d17c13ddc9 100644 --- a/shards/requirements.txt +++ b/shards/requirements.txt @@ -1,21 +1,41 @@ +astroid==2.3.3 +atomicwrites==1.3.0 +attrs==19.3.0 +cachetools==3.1.1 +certifi==2019.9.11 +chardet==3.0.4 environs==4.2.0 factory-boy==2.12.0 Faker==1.0.7 fire==0.1.3 google-auth==1.6.3 grpcio==1.22.0 +grpcio-opentracing==1.1.4 grpcio-tools==1.22.0 +idna==2.8 +importlib-metadata==0.23 +isort==4.3.21 +jaeger-client==4.1.0 kubernetes==10.0.1 +lazy-object-proxy==1.4.3 MarkupSafe==1.1.1 marshmallow==2.19.5 -pymysql==0.9.3 +mccabe==0.6.1 +mock==2.0.0 +more-itertools==7.2.0 +oauthlib==3.1.0 +opentracing==2.2.0 +packaging==19.2 +pbr==5.4.3 +pluggy==0.13.0 +pluginbase==1.0.0 protobuf==3.9.1 py==1.8.0 pyasn1==0.4.7 pyasn1-modules==0.2.6 pylint==2.3.1 -pymilvus-test==0.2.28 -#pymilvus==0.2.0 +pymilvus==0.2.5 +PyMySQL==0.9.3 pyparsing==2.4.0 pytest==4.6.3 pytest-level==0.1.1 @@ -25,13 +45,19 @@ pytest-timeout==1.3.3 python-dateutil==2.8.0 python-dotenv==0.10.3 pytz==2019.1 +PyYAML==5.1.2 requests==2.22.0 requests-oauthlib==1.2.0 rsa==4.0 six==1.12.0 SQLAlchemy==1.3.5 +text-unidecode==1.2 +threadloop==1.0.2 +thrift==0.13.0 +tornado==5.1.1 +typed-ast==1.4.0 urllib3==1.25.3 -jaeger-client>=3.4.0 -grpcio-opentracing>=1.0 -mock==2.0.0 -pluginbase==1.0.0 +wcwidth==0.1.7 +websocket-client==0.56.0 +wrapt==1.11.2 +zipp==0.6.0 From 692b2110315fc093ab908d44059875417ec5f4ec Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 11:51:23 +0800 Subject: [PATCH 48/56] update requirements only pymilvus --- shards/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shards/requirements.txt b/shards/requirements.txt index 14bdde2a06..4ea03e0b17 100644 --- a/shards/requirements.txt +++ b/shards/requirements.txt @@ -14,7 +14,7 @@ py==1.8.0 pyasn1==0.4.7 pyasn1-modules==0.2.6 pylint==2.3.1 -pymilvus-test==0.2.28 +pymilvus==0.2.5 #pymilvus==0.2.0 pyparsing==2.4.0 pytest==4.6.3 From 15dcce417e3c2b68785d09075f885378ef50a4bf Mon Sep 17 00:00:00 2001 From: groot Date: Wed, 20 Nov 2019 12:24:18 +0800 Subject: [PATCH 49/56] #377 Create partition success if tag name only contains spaces --- CHANGELOG.md | 12 ++++-- core/src/db/DBImpl.cpp | 11 ++++- core/src/db/meta/MySQLMetaImpl.cpp | 40 ++++++++++++++----- core/src/db/meta/SqliteMetaImpl.cpp | 39 +++++++++++++----- .../request/CreatePartitionRequest.cpp | 2 +- .../request/DropPartitionRequest.cpp | 38 +++++++++++++----- core/src/utils/ValidationUtil.cpp | 22 ++++++++-- core/src/utils/ValidationUtil.h | 3 ++ core/unittest/db/test_db.cpp | 7 ++++ core/unittest/db/test_db_mysql.cpp | 8 ++++ 10 files changed, 141 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ced0633cdc..dcd8c161d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,24 +12,28 @@ Please mark all change in change log and use the ticket from JIRA. - \#340 - Test cases run failed on 0.6.0 - \#353 - Rename config.h.in to version.h.in - \#374 - sdk_simple return empty result +- \#377 - Create partition success if tag name only contains spaces - \#397 - sdk_simple return incorrect result +- \#399 - Create partition should be failed if partition tag existed +- \#412 - Message returned is confused when partition created with null partition name +- \#416 - Drop the same partition success repeatally ## Feature - \#12 - Pure CPU version for Milvus - \#77 - Support table partition -- \#226 - Experimental shards middleware for Milvus - \#127 - Support new Index type IVFPQ +- \#226 - Experimental shards middleware for Milvus ## Improvement +- \#255 - Add ivfsq8 test report detailed version - \#275 - Rename C++ SDK IndexType -- \#284 - Change C++ SDK to shared library - \#260 - C++ SDK README - \#266 - Rpc request source code refactor -- \#314 - add Find FAISS in CMake +- \#284 - Change C++ SDK to shared library - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue +- \#314 - add Find FAISS in CMake - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md -- \#255 - Add ivfsq8 test report detailed version - \#404 - Add virtual method Init() in Pass abstract class - \#409 - Add a Fallback pass in optimizer diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 2559b3a46b..7b0103f52e 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -279,6 +279,11 @@ DBImpl::DropPartitionByTag(const std::string& table_id, const std::string& parti std::string partition_name; auto status = meta_ptr_->GetPartitionName(table_id, partition_tag, partition_name); + if (!status.ok()) { + ENGINE_LOG_ERROR << status.message(); + return status; + } + return DropPartition(partition_name); } @@ -853,8 +858,12 @@ DBImpl::GetPartitionsByTags(const std::string& table_id, const std::vectorShowPartitions(table_id, partiton_array); for (auto& tag : partition_tags) { + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); for (auto& schema : partiton_array) { - if (server::StringHelpFunctions::IsRegexMatch(schema.partition_tag_, tag)) { + if (server::StringHelpFunctions::IsRegexMatch(schema.partition_tag_, valid_tag)) { partition_name_array.insert(schema.table_id_); } } diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index bf83447806..df994af03f 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -22,6 +22,7 @@ #include "metrics/Metrics.h" #include "utils/Exception.h" #include "utils/Log.h" +#include "utils/StringHelpFunctions.h" #include #include @@ -1165,14 +1166,20 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p return Status(DB_ERROR, "Nested partition is not allow"); } - if (partition_name == "") { - // not allow duplicated partition - std::string exist_partition; - GetPartitionName(table_id, tag, exist_partition); - if (!exist_partition.empty()) { - return Status(DB_ERROR, "Duplicated partition is not allow"); - } + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + // not allow duplicated partition + std::string exist_partition; + GetPartitionName(table_id, valid_tag, exist_partition); + if (!exist_partition.empty()) { + return Status(DB_ERROR, "Duplicated partition is not allow"); + } + + if (partition_name == "") { + // generate unique partition name NextTableId(table_schema.table_id_); } else { table_schema.table_id_ = partition_name; @@ -1182,9 +1189,14 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p table_schema.flag_ = 0; table_schema.created_on_ = utils::GetMicroSecTimeStamp(); table_schema.owner_table_ = table_id; - table_schema.partition_tag_ = tag; + table_schema.partition_tag_ = valid_tag; - return CreateTable(table_schema); + status = CreateTable(table_schema); + if (status.code() == DB_ALREADY_EXIST) { + return Status(DB_ALREADY_EXIST, "Partition already exists"); + } + + return status; } Status @@ -1231,6 +1243,12 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& try { server::MetricCollector metric; mysqlpp::StoreQueryResult res; + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); @@ -1240,7 +1258,7 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& mysqlpp::Query allPartitionsQuery = connectionPtr->query(); allPartitionsQuery << "SELECT table_id FROM " << META_TABLES << " WHERE owner_table = " << mysqlpp::quote - << table_id << " AND partition_tag = " << mysqlpp::quote << tag << " AND state <> " + << table_id << " AND partition_tag = " << mysqlpp::quote << valid_tag << " AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allPartitionsQuery.str(); @@ -1252,7 +1270,7 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& const mysqlpp::Row& resRow = res[0]; resRow["table_id"].to_string(partition_name); } else { - return Status(DB_NOT_FOUND, "Partition " + tag + " of table " + table_id + " not found"); + return Status(DB_NOT_FOUND, "Partition " + valid_tag + " of table " + table_id + " not found"); } } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN GET PARTITION NAME", e.what()); diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 22e953fe9d..ffefc4ed43 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -22,6 +22,7 @@ #include "metrics/Metrics.h" #include "utils/Exception.h" #include "utils/Log.h" +#include "utils/StringHelpFunctions.h" #include #include @@ -760,14 +761,20 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& return Status(DB_ERROR, "Nested partition is not allow"); } - if (partition_name == "") { - // not allow duplicated partition - std::string exist_partition; - GetPartitionName(table_id, tag, exist_partition); - if(!exist_partition.empty()) { - return Status(DB_ERROR, "Duplicated partition is not allow"); - } + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + // not allow duplicated partition + std::string exist_partition; + GetPartitionName(table_id, valid_tag, exist_partition); + if(!exist_partition.empty()) { + return Status(DB_ERROR, "Duplicated partition is not allow"); + } + + if (partition_name == "") { + // generate unique partition name NextTableId(table_schema.table_id_); } else { table_schema.table_id_ = partition_name; @@ -777,9 +784,14 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& table_schema.flag_ = 0; table_schema.created_on_ = utils::GetMicroSecTimeStamp(); table_schema.owner_table_ = table_id; - table_schema.partition_tag_ = tag; + table_schema.partition_tag_ = valid_tag; - return CreateTable(table_schema); + status = CreateTable(table_schema); + if (status.code() == DB_ALREADY_EXIST) { + return Status(DB_ALREADY_EXIST, "Partition already exists"); + } + + return status; } Status @@ -814,13 +826,18 @@ SqliteMetaImpl::GetPartitionName(const std::string& table_id, const std::string& try { server::MetricCollector metric; + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + auto name = ConnectorPtr->select(columns(&TableSchema::table_id_), where(c(&TableSchema::owner_table_) == table_id - and c(&TableSchema::partition_tag_) == tag)); + and c(&TableSchema::partition_tag_) == valid_tag)); if (name.size() > 0) { partition_name = std::get<0>(name[0]); } else { - return Status(DB_NOT_FOUND, "Table " + table_id + "'s partition " + tag + " not found"); + return Status(DB_NOT_FOUND, "Table " + table_id + "'s partition " + valid_tag + " not found"); } } catch (std::exception &e) { return HandleException("Encounter exception when get partition name", e.what()); diff --git a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp index bc2e7dc105..3bd4a86ef6 100644 --- a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp +++ b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp @@ -51,7 +51,7 @@ CreatePartitionRequest::OnExecute() { return status; } - status = ValidationUtil::ValidateTableName(partition_param_->partition_name()); + status = ValidationUtil::ValidatePartitionName(partition_param_->partition_name()); if (!status.ok()) { return status; } diff --git a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp index 217ca63a2a..0e29b6abe8 100644 --- a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp +++ b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp @@ -22,6 +22,7 @@ #include "utils/ValidationUtil.h" #include +#include namespace milvus { namespace server { @@ -38,23 +39,40 @@ DropPartitionRequest::Create(const ::milvus::grpc::PartitionParam* partition_par Status DropPartitionRequest::OnExecute() { - if (!partition_param_->partition_name().empty()) { - auto status = ValidationUtil::ValidateTableName(partition_param_->partition_name()); - if (!status.ok()) { - return status; - } - return DBWrapper::DB()->DropPartition(partition_param_->partition_name()); - } else { - auto status = ValidationUtil::ValidateTableName(partition_param_->table_name()); + std::string table_name = partition_param_->table_name(); + std::string partition_name = partition_param_->partition_name(); + std::string partition_tag = partition_param_->tag(); + if (!partition_name.empty()) { + auto status = ValidationUtil::ValidateTableName(partition_name); if (!status.ok()) { return status; } - status = ValidationUtil::ValidatePartitionTags({partition_param_->tag()}); + // check partition existence + engine::meta::TableSchema table_info; + table_info.table_id_ = partition_name; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, + "Table " + table_name + "'s partition " + partition_name + " not found"); + } else { + return status; + } + } + + return DBWrapper::DB()->DropPartition(partition_name); + } else { + auto status = ValidationUtil::ValidateTableName(table_name); if (!status.ok()) { return status; } - return DBWrapper::DB()->DropPartitionByTag(partition_param_->table_name(), partition_param_->tag()); + + status = ValidationUtil::ValidatePartitionTags({partition_tag}); + if (!status.ok()) { + return status; + } + return DBWrapper::DB()->DropPartitionByTag(table_name, partition_tag); } } diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index ec696ff3e0..fc52eb32a9 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -18,6 +18,7 @@ #include "utils/ValidationUtil.h" #include "Log.h" #include "db/engine/ExecutionEngine.h" +#include "utils/StringHelpFunctions.h" #include #ifdef MILVUS_GPU_VERSION @@ -168,11 +169,26 @@ ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSc return Status::OK(); } +Status +ValidationUtil::ValidatePartitionName(const std::string& partition_name) { + if (partition_name.empty()) { + std::string msg = "Partition name should not be empty."; + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_TABLE_NAME, msg); + } + + return ValidateTableName(partition_name); +} + Status ValidationUtil::ValidatePartitionTags(const std::vector& partition_tags) { - for (auto& tag : partition_tags) { - if (tag.empty()) { - std::string msg = "Invalid partition tag: " + tag + ". " + "Partition tag should not be empty."; + for (const std::string& tag : partition_tags) { + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + StringHelpFunctions::TrimStringBlank(valid_tag); + if (valid_tag.empty()) { + std::string msg = "Invalid partition tag: " + valid_tag + ". " + "Partition tag should not be empty."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_NPROBE, msg); } diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 01801e295a..9157ffad56 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -55,6 +55,9 @@ class ValidationUtil { static Status ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); + static Status + ValidatePartitionName(const std::string& partition_name); + static Status ValidatePartitionTags(const std::vector& partition_tags); diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index d8614dd5d1..343e924e8e 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -461,6 +461,13 @@ TEST_F(DBTest, PARTITION_TEST) { stat = db_->CreatePartition(table_name, partition_name, partition_tag); ASSERT_TRUE(stat.ok()); + // not allow nested partition + stat = db_->CreatePartition(partition_name, "dumy", "dummy"); + ASSERT_FALSE(stat.ok()); + + // not allow duplicated partition + stat = db_->CreatePartition(table_name, partition_name, partition_tag); + ASSERT_FALSE(stat.ok()); std::vector xb; BuildVectors(INSERT_BATCH, xb); diff --git a/core/unittest/db/test_db_mysql.cpp b/core/unittest/db/test_db_mysql.cpp index f828431838..93eea2fd27 100644 --- a/core/unittest/db/test_db_mysql.cpp +++ b/core/unittest/db/test_db_mysql.cpp @@ -297,6 +297,14 @@ TEST_F(MySqlDBTest, PARTITION_TEST) { stat = db_->CreatePartition(table_name, partition_name, partition_tag); ASSERT_TRUE(stat.ok()); + // not allow nested partition + stat = db_->CreatePartition(partition_name, "dumy", "dummy"); + ASSERT_FALSE(stat.ok()); + + // not allow duplicated partition + stat = db_->CreatePartition(table_name, partition_name, partition_tag); + ASSERT_FALSE(stat.ok()); + std::vector xb; BuildVectors(INSERT_BATCH, xb); From b65685df8f30129f102cb3e9d30debfc7f4fe9d4 Mon Sep 17 00:00:00 2001 From: groot Date: Wed, 20 Nov 2019 14:15:57 +0800 Subject: [PATCH 50/56] fix typo --- core/src/db/meta/MySQLMetaImpl.cpp | 4 ++-- core/src/db/meta/SqliteMetaImpl.cpp | 4 ++-- core/unittest/server/test_rpc.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index df994af03f..4406b87f7e 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1163,7 +1163,7 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p // not allow create partition under partition if (!table_schema.owner_table_.empty()) { - return Status(DB_ERROR, "Nested partition is not allow"); + return Status(DB_ERROR, "Nested partition is not allowed"); } // trim side-blank of tag, only compare valid characters @@ -1175,7 +1175,7 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p std::string exist_partition; GetPartitionName(table_id, valid_tag, exist_partition); if (!exist_partition.empty()) { - return Status(DB_ERROR, "Duplicated partition is not allow"); + return Status(DB_ERROR, "Duplicate partition is not allowed"); } if (partition_name == "") { diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index ffefc4ed43..12128c074d 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -758,7 +758,7 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& // not allow create partition under partition if(!table_schema.owner_table_.empty()) { - return Status(DB_ERROR, "Nested partition is not allow"); + return Status(DB_ERROR, "Nested partition is not allowed"); } // trim side-blank of tag, only compare valid characters @@ -770,7 +770,7 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& std::string exist_partition; GetPartitionName(table_id, valid_tag, exist_partition); if(!exist_partition.empty()) { - return Status(DB_ERROR, "Duplicated partition is not allow"); + return Status(DB_ERROR, "Duplicate partition is not allowed"); } if (partition_name == "") { diff --git a/core/unittest/server/test_rpc.cpp b/core/unittest/server/test_rpc.cpp index 34d4b0ad3a..5753c68422 100644 --- a/core/unittest/server/test_rpc.cpp +++ b/core/unittest/server/test_rpc.cpp @@ -409,7 +409,7 @@ TEST_F(RpcHandlerTest, PARTITION_TEST) { partition_parm.set_partition_name(partition_name); handler->DropPartition(&context, &partition_parm, &response); - ASSERT_EQ(response.error_code(), ::grpc::Status::OK.error_code()); + ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code()); } TEST_F(RpcHandlerTest, CMD_TEST) { From 58e591ae7c1d24b929f7a542fead68fdae803e3c Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 20 Nov 2019 14:19:31 +0800 Subject: [PATCH 51/56] Update Jenkinsfile --- ci/jenkins/Jenkinsfile | 40 ++++++++++++++++++++-------------------- ci/scripts/build.sh | 10 ++++++++-- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 8d3953b112..bf7a70d11a 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -50,7 +50,7 @@ pipeline { } stages { - stage("Run Build") { + stage("Run GPU Version Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -60,7 +60,7 @@ pipeline { } stages { - stage('Build') { + stage('GPU Version Build') { steps { container('milvus-build-env') { script { @@ -69,7 +69,7 @@ pipeline { } } } - stage('Code Coverage') { + stage('GPU Version Code Coverage') { steps { container('milvus-build-env') { script { @@ -78,7 +78,7 @@ pipeline { } } } - stage('Upload Package') { + stage('Upload GPU Version Package') { steps { container('milvus-build-env') { script { @@ -90,7 +90,7 @@ pipeline { } } - stage("Publish docker images") { + stage("Publish GPU Version docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -100,7 +100,7 @@ pipeline { } stages { - stage('Publish') { + stage('Publish GPU Version') { steps { container('publish-images'){ script { @@ -112,7 +112,7 @@ pipeline { } } - stage("Deploy to Development") { + stage("Deploy GPU Version to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -122,7 +122,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage("Deploy GPU Version to Dev") { steps { container('milvus-test-env') { script { @@ -132,7 +132,7 @@ pipeline { } } - stage("Dev Test") { + stage("GPU Version Dev Test") { steps { container('milvus-test-env') { script { @@ -147,7 +147,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage ("Cleanup GPU Version Dev") { steps { container('milvus-test-env') { script { @@ -180,7 +180,7 @@ pipeline { } stages { - stage("Run Build") { + stage("Run CPU Version Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -190,7 +190,7 @@ pipeline { } stages { - stage('Build') { + stage('Build CPU Version') { steps { container('milvus-build-env') { script { @@ -199,7 +199,7 @@ pipeline { } } } - stage('Code Coverage') { + stage('CPU Version Code Coverage') { steps { container('milvus-build-env') { script { @@ -208,7 +208,7 @@ pipeline { } } } - stage('Upload Package') { + stage('Upload CPU Version Package') { steps { container('milvus-build-env') { script { @@ -220,7 +220,7 @@ pipeline { } } - stage("Publish docker images") { + stage("Publish CPU Version docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -230,7 +230,7 @@ pipeline { } stages { - stage('Publish') { + stage('Publish CPU Version') { steps { container('publish-images'){ script { @@ -242,7 +242,7 @@ pipeline { } } - stage("Deploy to Development") { + stage("Deploy CPU Version to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -252,7 +252,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage("Deploy CPU Version to Dev") { steps { container('milvus-test-env') { script { @@ -262,7 +262,7 @@ pipeline { } } - stage("Dev Test") { + stage("CPU Version Dev Test") { steps { container('milvus-test-env') { script { @@ -277,7 +277,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage ("Cleanup CPU Version Dev") { steps { container('milvus-test-env') { script { diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 27962ccb54..eb5205ad26 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -16,6 +16,7 @@ BUILD_TYPE="Debug" BUILD_UNITTEST="OFF" INSTALL_PREFIX="/opt/milvus" FAISS_ROOT="" +CUSTOMIZATION="OFF" # default use origin faiss BUILD_COVERAGE="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" @@ -23,7 +24,7 @@ GPU_VERSION="OFF" WITH_MKL="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "o:t:b:f:gulcjmh" arg +while getopts "o:t:b:f:gxulcjmh" arg do case $arg in o) @@ -41,6 +42,9 @@ do g) GPU_VERSION="ON"; ;; + x) + CUSTOMIZATION="ON"; + ;; u) echo "Build and run unittest cases" ; BUILD_UNITTEST="ON"; @@ -66,6 +70,7 @@ parameter: -b: core code build directory -f: faiss root path -g: gpu version +-x: milvus customization (default: OFF) -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) -c: code coverage(default: OFF) @@ -74,7 +79,7 @@ parameter: -h: help usage: -./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-u] [-l] [-c] [-j] [-m] [-h] +./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h] " exit 0 ;; @@ -96,6 +101,7 @@ CMAKE_CMD="cmake \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ +-DCUSTOMIZATION=${CUSTOMIZATION} \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ From c89bb6699ebcfdc1239635b7775bf3a827e0970b Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 11:24:54 +0800 Subject: [PATCH 52/56] #306 Use int64 for all config integer --- core/src/db/Options.h | 2 +- core/src/db/engine/ExecutionEngineImpl.cpp | 4 +- core/src/db/engine/ExecutionEngineImpl.h | 4 +- core/src/scheduler/SchedInst.cpp | 4 +- .../scheduler/optimizer/BuildIndexPass.cpp | 2 +- core/src/scheduler/optimizer/FaissFlatPass.h | 4 +- .../scheduler/optimizer/FaissIVFFlatPass.h | 4 +- .../scheduler/optimizer/FaissIVFSQ8HPass.cpp | 2 +- .../scheduler/optimizer/FaissIVFSQ8HPass.h | 4 +- .../src/scheduler/optimizer/FaissIVFSQ8Pass.h | 4 +- core/src/server/Config.cpp | 70 +++++++++---------- core/src/server/Config.h | 16 ++--- core/src/server/DBWrapper.cpp | 8 +-- core/src/utils/CommonUtil.cpp | 2 +- core/src/utils/CommonUtil.h | 2 +- core/src/wrapper/KnowhereResource.cpp | 4 +- core/unittest/server/test_config.cpp | 45 ++++++------ core/unittest/server/test_util.cpp | 2 +- 18 files changed, 91 insertions(+), 92 deletions(-) diff --git a/core/src/db/Options.h b/core/src/db/Options.h index ebecb4de5a..91459a8d5f 100644 --- a/core/src/db/Options.h +++ b/core/src/db/Options.h @@ -33,7 +33,7 @@ static const char* ARCHIVE_CONF_DISK = "disk"; static const char* ARCHIVE_CONF_DAYS = "days"; struct ArchiveConf { - using CriteriaT = std::map; + using CriteriaT = std::map; explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string()); diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 4f2f542254..973d8b5912 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -146,7 +146,7 @@ ExecutionEngineImpl::HybridLoad() const { const std::string key = location_ + ".quantizer"; server::Config& config = server::Config::GetInstance(); - std::vector gpus; + std::vector gpus; Status s = config.GetGpuResourceConfigSearchResources(gpus); if (!s.ok()) { ENGINE_LOG_ERROR << s.message(); @@ -586,7 +586,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); - std::vector gpu_ids; + std::vector gpu_ids; Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids); for (auto id : gpu_ids) { if (gpu_num_ == id) { diff --git a/core/src/db/engine/ExecutionEngineImpl.h b/core/src/db/engine/ExecutionEngineImpl.h index 7eb304426e..da0e7cfb64 100644 --- a/core/src/db/engine/ExecutionEngineImpl.h +++ b/core/src/db/engine/ExecutionEngineImpl.h @@ -122,8 +122,8 @@ class ExecutionEngineImpl : public ExecutionEngine { int64_t dim_; std::string location_; - int32_t nlist_ = 0; - int32_t gpu_num_ = 0; + int64_t nlist_ = 0; + int64_t gpu_num_ = 0; }; } // namespace engine diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 867052ef5d..69d293f986 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -55,9 +55,9 @@ load_simple_config() { // get resources #ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); - std::vector gpu_ids; + std::vector gpu_ids; config.GetGpuResourceConfigSearchResources(gpu_ids); - std::vector build_gpu_ids; + std::vector build_gpu_ids; config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids); auto pcie = Connection("pcie", 12000); diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index cd1b7550d8..d535b9675f 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -26,7 +26,7 @@ namespace scheduler { void BuildIndexPass::Init() { server::Config& config = server::Config::GetInstance(); - std::vector build_resources; + std::vector build_resources; Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); if (!s.ok()) { throw; diff --git a/core/src/scheduler/optimizer/FaissFlatPass.h b/core/src/scheduler/optimizer/FaissFlatPass.h index 60adf62a07..f219bebdf3 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.h +++ b/core/src/scheduler/optimizer/FaissFlatPass.h @@ -45,9 +45,9 @@ class FaissFlatPass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissFlatPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.h b/core/src/scheduler/optimizer/FaissIVFFlatPass.h index 9264db32a8..2d15539014 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.h +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.h @@ -45,9 +45,9 @@ class FaissIVFFlatPass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissIVFFlatPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index b63a0f7490..a99e861e03 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -32,7 +32,7 @@ FaissIVFSQ8HPass::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { - threshold_ = std::numeric_limits::max(); + threshold_ = std::numeric_limits::max(); } s = config.GetGpuResourceConfigSearchResources(gpus); } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h index fa830ff08a..0d2892809f 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h @@ -45,9 +45,9 @@ class FaissIVFSQ8HPass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissIVFSQ8HPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h index c5853283f7..e92ea2fe4e 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h @@ -45,9 +45,9 @@ class FaissIVFSQ8Pass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissIVFSQ8PassPtr = std::shared_ptr; diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 2ff91bfe3a..f3efcff0cc 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -113,19 +113,19 @@ Config::ValidateConfig() { return s; } - int32_t db_archive_disk_threshold; + int64_t db_archive_disk_threshold; s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); if (!s.ok()) { return s; } - int32_t db_archive_days_threshold; + int64_t db_archive_days_threshold; s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); if (!s.ok()) { return s; } - int32_t db_insert_buffer_size; + int64_t db_insert_buffer_size; s = GetDBConfigInsertBufferSize(db_insert_buffer_size); if (!s.ok()) { return s; @@ -170,19 +170,19 @@ Config::ValidateConfig() { } /* engine config */ - int32_t engine_use_blas_threshold; + int64_t engine_use_blas_threshold; s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); if (!s.ok()) { return s; } - int32_t engine_omp_thread_num; + int64_t engine_omp_thread_num; s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); if (!s.ok()) { return s; } - int32_t engine_gpu_search_threshold; + int64_t engine_gpu_search_threshold; s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold); if (!s.ok()) { return s; @@ -208,13 +208,13 @@ Config::ValidateConfig() { return s; } - std::vector search_resources; + std::vector search_resources; s = GetGpuResourceConfigSearchResources(search_resources); if (!s.ok()) { return s; } - std::vector index_build_resources; + std::vector index_build_resources; s = GetGpuResourceConfigBuildIndexResources(index_build_resources); if (!s.ok()) { return s; @@ -485,7 +485,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { ". Possible reason: db_config.insert_buffer_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t buffer_size = std::stoi(value) * GB; + int64_t buffer_size = std::stoll(value) * GB; if (buffer_size <= 0) { std::string msg = "Invalid insert buffer size: " + value + ". Possible reason: db_config.insert_buffer_size is not a positive integer."; @@ -540,7 +540,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t cpu_cache_capacity = std::stoi(value) * GB; + int64_t cpu_cache_capacity = std::stoll(value) * GB; if (cpu_cache_capacity <= 0) { std::string msg = "Invalid cpu cache capacity: " + value + ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; @@ -557,7 +557,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; } - int32_t buffer_value; + int64_t buffer_value; Status s = GetDBConfigInsertBufferSize(buffer_value); if (!s.ok()) { return s; @@ -619,10 +619,10 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } - int32_t omp_thread = std::stoi(value); - uint32_t sys_thread_cnt = 8; + int64_t omp_thread = std::stoll(value); + int64_t sys_thread_cnt = 8; CommonUtil::GetSystemAvailableThreads(sys_thread_cnt); - if (omp_thread > static_cast(sys_thread_cnt)) { + if (omp_thread > sys_thread_cnt) { std::string msg = "Invalid omp thread num: " + value + ". Possible reason: engine_config.omp_thread_num exceeds system cpu cores."; return Status(SERVER_INVALID_ARGUMENT, msg); @@ -657,14 +657,14 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { ". Possible reason: gpu_resource_config.cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - uint64_t gpu_cache_capacity = std::stoi(value) * GB; - std::vector gpu_ids; + int64_t gpu_cache_capacity = std::stoll(value) * GB; + std::vector gpu_ids; Status s = GetGpuResourceConfigBuildIndexResources(gpu_ids); if (!s.ok()) { return s; } - for (int32_t gpu_id : gpu_ids) { + for (int64_t gpu_id : gpu_ids) { size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) { std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id); @@ -855,37 +855,37 @@ Config::GetDBConfigBackendUrl(std::string& value) { } Status -Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { +Config::GetDBConfigArchiveDiskThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { +Config::GetDBConfigArchiveDaysThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigInsertBufferSize(int32_t& value) { +Config::GetDBConfigInsertBufferSize(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -927,7 +927,7 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -957,37 +957,37 @@ Config::GetCacheConfigCacheInsertData(bool& value) { } Status -Config::GetEngineConfigUseBlasThreshold(int32_t& value) { +Config::GetEngineConfigUseBlasThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); Status s = CheckEngineConfigUseBlasThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigOmpThreadNum(int32_t& value) { +Config::GetEngineConfigOmpThreadNum(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { +Config::GetEngineConfigGpuSearchThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); Status s = CheckEngineConfigGpuSearchThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -1020,7 +1020,7 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -1046,7 +1046,7 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) { } Status -Config::GetGpuResourceConfigSearchResources(std::vector& value) { +Config::GetGpuResourceConfigSearchResources(std::vector& value) { bool gpu_resource_enable = false; Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { @@ -1065,13 +1065,13 @@ Config::GetGpuResourceConfigSearchResources(std::vector& value) { return s; } for (std::string& res : res_vec) { - value.push_back(std::stoi(res.substr(3))); + value.push_back(std::stoll(res.substr(3))); } return Status::OK(); } Status -Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { +Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { bool gpu_resource_enable = false; Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { @@ -1091,7 +1091,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { return s; } for (std::string& res : res_vec) { - value.push_back(std::stoi(res.substr(3))); + value.push_back(std::stoll(res.substr(3))); } return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index a6e78671b3..0907080a6f 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -210,11 +210,11 @@ class Config { Status GetDBConfigBackendUrl(std::string& value); Status - GetDBConfigArchiveDiskThreshold(int32_t& value); + GetDBConfigArchiveDiskThreshold(int64_t& value); Status - GetDBConfigArchiveDaysThreshold(int32_t& value); + GetDBConfigArchiveDaysThreshold(int64_t& value); Status - GetDBConfigInsertBufferSize(int32_t& value); + GetDBConfigInsertBufferSize(int64_t& value); Status GetDBConfigPreloadTable(std::string& value); @@ -236,11 +236,11 @@ class Config { /* engine config */ Status - GetEngineConfigUseBlasThreshold(int32_t& value); + GetEngineConfigUseBlasThreshold(int64_t& value); Status - GetEngineConfigOmpThreadNum(int32_t& value); + GetEngineConfigOmpThreadNum(int64_t& value); Status - GetEngineConfigGpuSearchThreshold(int32_t& value); + GetEngineConfigGpuSearchThreshold(int64_t& value); /* gpu resource config */ Status @@ -250,9 +250,9 @@ class Config { Status GetGpuResourceConfigCacheThreshold(float& value); Status - GetGpuResourceConfigSearchResources(std::vector& value); + GetGpuResourceConfigSearchResources(std::vector& value); Status - GetGpuResourceConfigBuildIndexResources(std::vector& value); + GetGpuResourceConfigBuildIndexResources(std::vector& value); public: /* server config */ diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index 1ff914dc69..f7a43d671c 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -89,7 +89,7 @@ DBWrapper::StartService() { } // engine config - int32_t omp_thread; + int64_t omp_thread; s = config.GetEngineConfigOmpThreadNum(omp_thread); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -100,7 +100,7 @@ DBWrapper::StartService() { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; } else { - uint32_t sys_thread_cnt = 8; + int64_t sys_thread_cnt = 8; if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { omp_thread = static_cast(ceil(sys_thread_cnt * 0.5)); omp_set_num_threads(omp_thread); @@ -108,7 +108,7 @@ DBWrapper::StartService() { } // init faiss global variable - int32_t use_blas_threshold; + int64_t use_blas_threshold; s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -119,7 +119,7 @@ DBWrapper::StartService() { // set archive config engine::ArchiveConf::CriteriaT criterial; - int32_t disk, days; + int64_t disk, days; s = config.GetDBConfigArchiveDiskThreshold(disk); if (!s.ok()) { std::cerr << s.ToString() << std::endl; diff --git a/core/src/utils/CommonUtil.cpp b/core/src/utils/CommonUtil.cpp index fbf3112aeb..26e43619fb 100644 --- a/core/src/utils/CommonUtil.cpp +++ b/core/src/utils/CommonUtil.cpp @@ -54,7 +54,7 @@ CommonUtil::GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem) { } bool -CommonUtil::GetSystemAvailableThreads(uint32_t& thread_count) { +CommonUtil::GetSystemAvailableThreads(int64_t& thread_count) { // threadCnt = std::thread::hardware_concurrency(); thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count *= THREAD_MULTIPLY_CPU; diff --git a/core/src/utils/CommonUtil.h b/core/src/utils/CommonUtil.h index 939bdd6d31..121196986a 100644 --- a/core/src/utils/CommonUtil.h +++ b/core/src/utils/CommonUtil.h @@ -30,7 +30,7 @@ class CommonUtil { static bool GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem); static bool - GetSystemAvailableThreads(uint32_t& thread_count); + GetSystemAvailableThreads(int64_t& thread_count); static bool IsFileExist(const std::string& path); diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index bf6675cafa..5a2296b16e 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -48,7 +48,7 @@ KnowhereResource::Initialize() { // get build index gpu resource server::Config& config = server::Config::GetInstance(); - std::vector build_index_gpus; + std::vector build_index_gpus; s = config.GetGpuResourceConfigBuildIndexResources(build_index_gpus); if (!s.ok()) return s; @@ -58,7 +58,7 @@ KnowhereResource::Initialize() { } // get search gpu resource - std::vector search_gpus; + std::vector search_gpus; s = config.GetGpuResourceConfigSearchResources(search_gpus); if (!s.ok()) return s; diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index e0fde62916..664a08d631 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -104,7 +104,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { milvus::server::Config& config = milvus::server::Config::GetInstance(); milvus::Status s; std::string str_val; - int32_t int32_val; int64_t int64_val; float float_val; bool bool_val; @@ -160,26 +159,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(s.ok()); ASSERT_TRUE(str_val == db_backend_url); - int32_t db_archive_disk_threshold = 100; + int64_t db_archive_disk_threshold = 100; s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDiskThreshold(int32_val); + s = config.GetDBConfigArchiveDiskThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_archive_disk_threshold); + ASSERT_TRUE(int64_val == db_archive_disk_threshold); - int32_t db_archive_days_threshold = 365; + int64_t db_archive_days_threshold = 365; s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDaysThreshold(int32_val); + s = config.GetDBConfigArchiveDaysThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_archive_days_threshold); + ASSERT_TRUE(int64_val == db_archive_days_threshold); - int32_t db_insert_buffer_size = 2; + int64_t db_insert_buffer_size = 2; s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigInsertBufferSize(int32_val); + s = config.GetDBConfigInsertBufferSize(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_insert_buffer_size); + ASSERT_TRUE(int64_val == db_insert_buffer_size); /* metric config */ bool metric_enable_monitor = false; @@ -223,26 +222,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(bool_val == cache_insert_data); /* engine config */ - int32_t engine_use_blas_threshold = 50; + int64_t engine_use_blas_threshold = 50; s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigUseBlasThreshold(int32_val); + s = config.GetEngineConfigUseBlasThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_use_blas_threshold); + ASSERT_TRUE(int64_val == engine_use_blas_threshold); - int32_t engine_omp_thread_num = 8; + int64_t engine_omp_thread_num = 8; s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigOmpThreadNum(int32_val); + s = config.GetEngineConfigOmpThreadNum(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_omp_thread_num); + ASSERT_TRUE(int64_val == engine_omp_thread_num); - int32_t engine_gpu_search_threshold = 800; + int64_t engine_gpu_search_threshold = 800; s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigGpuSearchThreshold(int32_val); + s = config.GetEngineConfigGpuSearchThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_gpu_search_threshold); + ASSERT_TRUE(int64_val == engine_gpu_search_threshold); /* gpu resource config */ bool resource_enable_gpu = true; @@ -267,7 +266,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(float_val == gpu_cache_threshold); std::vector search_resources = {"gpu0"}; - std::vector search_res_vec; + std::vector search_res_vec; std::string search_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( search_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, search_res_str); @@ -276,11 +275,11 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.GetGpuResourceConfigSearchResources(search_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < search_resources.size(); i++) { - ASSERT_TRUE(std::stoi(search_resources[i].substr(3)) == search_res_vec[i]); + ASSERT_TRUE(std::stoll(search_resources[i].substr(3)) == search_res_vec[i]); } std::vector build_index_resources = {"gpu0"}; - std::vector build_index_res_vec; + std::vector build_index_res_vec; std::string build_index_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( build_index_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, build_index_res_str); @@ -289,7 +288,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.GetGpuResourceConfigBuildIndexResources(build_index_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < build_index_resources.size(); i++) { - ASSERT_TRUE(std::stoi(build_index_resources[i].substr(3)) == build_index_res_vec[i]); + ASSERT_TRUE(std::stoll(build_index_resources[i].substr(3)) == build_index_res_vec[i]); } #endif } diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index 68400f2454..e5884cac65 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -60,7 +60,7 @@ TEST(UtilTest, COMMON_TEST) { ASSERT_GT(total_mem, 0); ASSERT_GT(free_mem, 0); - uint32_t thread_cnt = 0; + int64_t thread_cnt = 0; milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt); ASSERT_GT(thread_cnt, 0); From 82e6b2738b9b72af25b1495ff50a697cff2c5b1b Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 14:35:10 +0800 Subject: [PATCH 53/56] #306 update changelog --- CHANGELOG.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42d5b8978f..7d88bc270e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,15 +22,16 @@ Please mark all change in change log and use the ticket from JIRA. - \#346 - Support build index with multiple gpu ## Improvement -- \#275 - Rename C++ SDK IndexType -- \#284 - Change C++ SDK to shared library +- \#255 - Add ivfsq8 test report detailed version - \#260 - C++ SDK README - \#266 - Rpc request source code refactor -- \#314 - add Find FAISS in CMake +- \#275 - Rename C++ SDK IndexType +- \#284 - Change C++ SDK to shared library +- \#306 - Use int64 for all config integer - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue +- \#314 - add Find FAISS in CMake - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md -- \#255 - Add ivfsq8 test report detailed version - \#404 - Add virtual method Init() in Pass abstract class - \#409 - Add a Fallback pass in optimizer From 01cd2b1e022ffd861c0add9d9ad9c2dfdd38d13d Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 15:01:43 +0800 Subject: [PATCH 54/56] [skip ci] Add README for docs file --- docs/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 docs/README.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..64409d8240 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,11 @@ +# Docs + +This repository contains test reports on the search performance of different index types on standalone Milvus. + +The tests are run on [SIFT1B dataset](http://corpus-texmex.irisa.fr/), and provide results on the following measures: + +- Query Elapsed Time: Time cost (in seconds) to run a query. + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved. + +Test variables are `nq` and `topk`. From 94d11b99310599a0134c851194460e183013c8d6 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Wed, 20 Nov 2019 15:03:44 +0800 Subject: [PATCH 55/56] [skip ci] add ivfsq8h test report detailed version #255 --- ...us_ivfsq8h_test_report_detailed_version.md | 179 +++++++++++++++++ ...ivfsq8h_test_report_detailed_version_cn.md | 180 ++++++++++++++++++ 2 files changed, 359 insertions(+) create mode 100755 docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md create mode 100644 docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md new file mode 100755 index 0000000000..042d059684 --- /dev/null +++ b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md @@ -0,0 +1,179 @@ +# milvus_ivfsq8h_test_report_detailed_version + +## Summary + +This document contains the test reports of IVF_SQ8H index on Milvus single server. + + + +## Test objectives + +The time cost and recall when searching with different parameters. + + + +## Test method + +### Hardware/Software requirements + +Operating System: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +Memory: 503GB + +Docker version: 18.09 + +NVIDIA Driver version: 430.34 + +Milvus version: 0.5.3 + +SDK interface: Python 3.6.8 + +pymilvus version: 0.2.5 + + + +### Data model + +The data used in the tests are: + +- Data source: sift1b +- Data type: hdf5 + +For details on this dataset, please check : http://corpus-texmex.irisa.fr/ . + + + +### Measures + +- Query Elapsed Time: Time cost (in seconds) to run a query. Variables that affect Query Elapsed Time: + + - nq (Number of queried vectors) + +> Note: In the query test of query elapsed time, we will test the following parameters with different values: + > + > nq - grouped by: [1, 5, 10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]. + > + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved . Variables that affect Recall: + + - nq (Number of queried vectors) + - topk (Top k result of a query) + + > Note: In the query test of recall, we will test the following parameters with different values: + > + > nq - grouped by: [10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800], + > + > topk - grouped by: [1, 10, 100] + + + +## Test reports + +### Test environment + +Data base: sift1b-1,000,000,000 vectors, 128-dimension + +Table Attributes + +- nlist: 16384 +- metric_type: L2 + +Query configuration + +- nprobe: 32 + +Milvus configuration + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 +- gpu_search_threshold: 1200 +- search_resources: cpu, gpu0, gpu1 + +The definitions of Milvus configuration are on https://milvus.io/docs/en/reference/milvus_config/. + +Test method + +Test the query elapsed time and recall with several parameters, and once only change one parameter. + +- Whether to restart Milvus after each query: No + + + +### Performance test + +#### Data query + +**Test result** + +Query Elapsed Time + +topk = 100 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 0.34 | +| nq=5 | 0.72 | +| nq=10 | 0.91 | +| nq=50 | 1.51 | +| nq=100 | 2.49 | +| nq=200 | 4.09 | +| nq=400 | 7.32 | +| nq=600 | 10.63 | +| nq=800 | 13.84 | +| nq=1000 | 16.83 | +| nq=1200 | 18.20 | +| nq=1400 | 20.1 | +| nq=1600 | 20.0 | +| nq=1800 | 19.86 | + +When nq is 1800, the query time cost of a 128-dimension vector is around 11ms. + + + +**Conclusion** + +When nq < 1200, the query elapsed time increases quickly with nq; when nq > 1200, the query elapsed time increases much slower. It is because gpu_search_threshold is set to 1200, when nq < 1200, CPU is chosen to do the query, otherwise GPU is chosen. Compared with CPU, GPU has much more cores and stronger computing capability. When nq is large, it can better reflect GPU's advantages on computing. + +The query elapsed time consists of two parts: (1) index CPU-to-GPU copy time; (2) nprobe buckets search time. When nq is larger enough, index CPU-to-GPU copy time can be amortized efficiently. So Milvus performs well through setting suitable gpu_search_threshold. + + + +### Recall test + +**Test result** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +We use the ground_truth in sift1b dataset to calculate the recall of query results. + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=50 | 0.980 | 0.950 | 0.941 | +| nq=100 | 0.970 | 0.937 | 0.931 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | +| nq=1200 | 0.937 | 0.943 | 0.931 | +| nq=1400 | 0.939 | 0.945 | 0.931 | +| nq=1600 | 0.936 | 0.945 | 0.931 | +| nq=1800 | 0.937 | 0.946 | 0.932 | + + + +**Conclusion** + +As nq increases, the recall gradually stabilizes to over 93%. The usage of CPU or GPU and different topk are not related to recall. \ No newline at end of file diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md new file mode 100644 index 0000000000..b50d00f9bd --- /dev/null +++ b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md @@ -0,0 +1,180 @@ +# milvus_ivfsq8h_test_report_detailed_version_cn + +## 概述 + +本文描述了ivfsq8h索引在milvus单机部署方式下的测试结果。 + + + +## 测试目标 + +参数不同情况下的查询时间和召回率。 + + + +## 测试方法 + +### 软硬件环境 + +操作系统：CentOS Linux release 7.6.1810 (Core) + +CPU：Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0：GeForce GTX 1080 + +GPU1：GeForce GTX 1080 + +内存：503GB + +Docker版本：18.09 + +NVIDIA Driver版本：430.34 + +Milvus版本：0.5.3 + +SDK接口：Python 3.6.8 + +pymilvus版本：0.2.5 + + + +### 数据模型 + +本测试中用到的主要数据: + +- 数据来源：sift1b +- 数据类型：hdf5 + +关于该数据集的详细信息请参考：http://corpus-texmex.irisa.fr/ 。 + + + +### 测试指标 + +- Query Elapsed Time: 数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量： + + - nq (被查询向量的数量) + +> 备注：在向量查询测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [1, 5, 10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]的数量分组。 + > + +- Recall: 实际返回的正确结果占总数之比。影响Recall的变量： + + - nq (被查询向量的数量) + - topk (单条查询中最相似的K个结果) + + > 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]的数量分组， + > + > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。 + + + +## 测试报告 + +### 测试环境 + +数据集: sift1b-1,000,000,000向量, 128维 + +表格属性： + +- nlist: 16384 +- metric_type: L2 + +查询设置： + +- nprobe: 32 + +Milvus设置： + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 +- gpu_search_threshold: 1200 +- search_resources: cpu, gpu0, gpu1 + +Milvus设置的详细定义可以参考 https://milvus.io/docs/en/reference/milvus_config/ 。 + +测试方法 + +通过一次仅改变一个参数的值，测试查询向量时间和召回率。 + +- 查询后是否重启Milvus：否 + + + +### 性能测试 + +#### 数据查询 + +测试结果 + +Query Elapsed Time + +topk = 100 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 0.34 | +| nq=5 | 0.72 | +| nq=10 | 0.91 | +| nq=50 | 1.51 | +| nq=100 | 2.49 | +| nq=200 | 4.09 | +| nq=400 | 7.32 | +| nq=600 | 10.63 | +| nq=800 | 13.84 | +| nq=1000 | 16.83 | +| nq=1200 | 18.20 | +| nq=1400 | 20.1 | +| nq=1600 | 20.0 | +| nq=1800 | 19.86 | + +当nq为1800时，查询一条128维向量需要耗时约11毫秒。 + + + +**总结** + +当nq小于1200时，查询耗时随nq的增长快速增大；当nq大于1200时，查询耗时的增大则缓慢许多。这是因为gpu_search_threshold这一参数的值被设为1200，当nq<1200时，选择CPU进行操作，否则选择GPU进行操作。与CPU。 + +在GPU模式下的查询耗时由两部分组成：（1）索引从CPU到GPU的拷贝时间；（2）所有分桶的查询时间。当nq小于500时，索引从CPU到GPU 的拷贝时间无法被有效均摊，此时CPU模式时一个更优的选择；当nq大于500时，选择GPU模式更合理。和CPU相比，GPU具有更多的核数和更强的算力。当nq较大时，GPU在计算上的优势能被更好地被体现。 + + + +### 召回率测试 + +**测试结果** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +我们利用sift1b数据集中的ground_truth来计算查询结果的召回率。 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=50 | 0.980 | 0.950 | 0.941 | +| nq=100 | 0.970 | 0.937 | 0.931 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | +| nq=1200 | 0.937 | 0.943 | 0.931 | +| nq=1400 | 0.939 | 0.945 | 0.931 | +| nq=1600 | 0.936 | 0.945 | 0.931 | +| nq=1800 | 0.937 | 0.946 | 0.932 | + + + +**总结** + +随着nq的增大，召回率逐渐稳定至93%以上。CPU/GPU的使用以及topk的值与召回率的大小无关。 + From a77bd78924fb6fe0d6be82825ba79dbfaa7f969e Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Wed, 20 Nov 2019 16:05:14 +0800 Subject: [PATCH 56/56] Fix compiling error --- core/src/wrapper/VecImpl.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index 2085daed24..9918a9a025 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -21,11 +21,13 @@ #include "knowhere/index/vector_index/IndexIDMAP.h" #include "utils/Log.h" #include "wrapper/WrapperException.h" +#include "wrapper/gpu/GPUVecImpl.h" #ifdef MILVUS_GPU_VERSION -#include -#include +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" +#include "knowhere/index/vector_index/IndexIVFSQHybrid.h" #endif