From bc93735b18ca160394fc02654b868a1bf1094bb5 Mon Sep 17 00:00:00 2001 From: erdustiggen Date: Thu, 14 Nov 2019 18:48:02 +0100 Subject: [PATCH 01/82] Changed from std::cout to LOG for error messages --- core/src/scheduler/task/BuildIndexTask.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index d8602c141e..daab7c9dec 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -146,8 +146,7 @@ XBuildIndexTask::Execute() { status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" - << std::endl; + ENGINE_LOG_ERROR << "Failed to build index, index file is too large or gpu memory is not enough"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); @@ -179,8 +178,8 @@ XBuildIndexTask::Execute() { status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to persist index file: " << table_file.location_ - << ", possible out of disk space" << std::endl; + ENGINE_LOG_ERROR << "Failed to persist index file: " << table_file.location_ + << ", possible out of disk space"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); From d53385caa7b90eab2bb851d597f87bbe2c42cd43 Mon Sep 17 00:00:00 2001 From: erdustiggen Date: Thu, 14 Nov 2019 20:54:25 +0100 Subject: [PATCH 02/82] clang format fix --- core/src/scheduler/task/BuildIndexTask.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index daab7c9dec..f561fa947d 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -179,7 +179,7 @@ XBuildIndexTask::Execute() { ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; ENGINE_LOG_ERROR << "Failed to persist index file: " << table_file.location_ - << ", possible out of disk space"; + << ", possible out of disk space"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); From 1159880036a01b1b2750889592ad2826b31a470a Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Fri, 15 Nov 2019 17:00:53 +0800 Subject: [PATCH 03/82] ignore proto python --- core/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/.gitignore b/core/.gitignore index 74e41dba6b..8db8df41db 100644 --- a/core/.gitignore +++ b/core/.gitignore @@ -9,3 +9,5 @@ output.info output_new.info server.info *.pyc +src/grpc/python_gen.h +src/grpc/python/ From a7cb907e44e44481e7391a0f08c2654359430e77 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Fri, 15 Nov 2019 20:15:34 +0800 Subject: [PATCH 04/82] remove gpu in unittest --- core/unittest/db/utils.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index f87371fe6b..bcf1462ecd 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -155,12 +155,10 @@ DBTest::SetUp() { res_mgr->Clear(); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - res_mgr->Add(milvus::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); auto default_conn = milvus::scheduler::Connection("IO", 500.0); auto PCIE = milvus::scheduler::Connection("IO", 11000.0); res_mgr->Connect("disk", "cpu", default_conn); - res_mgr->Connect("cpu", "gtx1660", PCIE); res_mgr->Start(); milvus::scheduler::SchedInst::GetInstance()->Start(); From 15315158e790d2be24ba45696bd27b27705aff7f Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Sat, 16 Nov 2019 09:34:17 +0800 Subject: [PATCH 05/82] modify sdk --- .../sdk/examples/partition/src/ClientTest.cpp | 65 ++++++++++--------- core/src/sdk/examples/simple/main.cpp | 2 +- .../sdk/examples/simple/src/ClientTest.cpp | 3 +- core/src/sdk/grpc/ClientProxy.cpp | 30 ++++++--- core/src/sdk/grpc/ClientProxy.h | 2 +- 5 files changed, 58 insertions(+), 44 deletions(-) diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index 775e1f6d60..b67b1fe907 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -31,45 +31,45 @@ namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); + const char *TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; -constexpr int64_t BATCH_ROW_COUNT = 10000; -constexpr int64_t NQ = 5; -constexpr int64_t TOP_K = 10; -constexpr int64_t NPROBE = 32; -constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different -constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; -constexpr int32_t N_LIST = 15000; -constexpr int32_t PARTITION_COUNT = 5; -constexpr int32_t TARGET_PARTITION = 3; + constexpr int64_t TABLE_DIMENSION = 512; + constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; + constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; + constexpr int64_t BATCH_ROW_COUNT = 10000; + constexpr int64_t NQ = 5; + constexpr int64_t TOP_K = 10; + constexpr int64_t NPROBE = 32; + constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different + constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; + constexpr int32_t N_LIST = 15000; + constexpr int32_t PARTITION_COUNT = 5; + constexpr int32_t TARGET_PARTITION = 3; -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; -} + milvus::TableSchema + BuildTableSchema() { + milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; + return tb_schema; + } -milvus::PartitionParam -BuildPartitionParam(int32_t index) { - std::string tag = std::to_string(index); - std::string partition_name = std::string(TABLE_NAME) + "_" + tag; - milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; - return partition_param; -} + milvus::PartitionParam + BuildPartitionParam(int32_t index) { + std::string tag = std::to_string(index); + std::string partition_name = std::string(TABLE_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; + return partition_param; + } -milvus::IndexParam -BuildIndexParam() { - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; - return index_param; -} + milvus::IndexParam + BuildIndexParam() { + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + return index_param; + } } // namespace void -ClientTest::Test(const std::string& address, const std::string& port) { +ClientTest::Test(const std::string &address, const std::string &port) { std::shared_ptr conn = milvus::Connection::Create(); milvus::Status stat; @@ -78,7 +78,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - +#ifdef yhz { // create table milvus::TableSchema tb_schema = BuildTableSchema(); stat = conn->CreateTable(tb_schema); @@ -202,4 +202,5 @@ ClientTest::Test(const std::string& address, const std::string& port) { } milvus::Connection::Destroy(conn); +#endif } diff --git a/core/src/sdk/examples/simple/main.cpp b/core/src/sdk/examples/simple/main.cpp index c08741606c..d9b6194329 100644 --- a/core/src/sdk/examples/simple/main.cpp +++ b/core/src/sdk/examples/simple/main.cpp @@ -36,7 +36,7 @@ main(int argc, char* argv[]) { {nullptr, 0, nullptr, 0}}; int option_index = 0; - std::string address = "127.0.0.1", port = "19530"; + std::string address = "192.168.1.89", port = "19530"; app_name = argv[0]; int value; diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp index dfa5e2219e..da43f2f2aa 100644 --- a/core/src/sdk/examples/simple/src/ClientTest.cpp +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -68,7 +68,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - +#ifdef yhz { // server version std::string version = conn->ServerVersion(); std::cout << "Server version: " << version << std::endl; @@ -206,4 +206,5 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; } +#endif } diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 4ec94cfa98..5e22904a08 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -43,16 +43,28 @@ Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; - channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - if (channel_ != nullptr) { - connected_ = true; - client_ptr_ = std::make_shared(channel_); - return Status::OK(); - } +// channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - std::string reason = "connect failed!"; - connected_ = false; - return Status(StatusCode::NotConnected, reason); +// channel_ = std::make_shared(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); +// channel_ = std::shared_ptr(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); + auto uri_str = uri.c_str(); + grpc_channel * channel = grpc_insecure_channel_create(uri_str, nullptr, nullptr); +// grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr); + auto state = grpc_channel_check_connectivity_state(channel, true); + if (state == GRPC_CHANNEL_READY) { + std::cout << "Connect " << uri << " successfully"; + } else { + std::cout << "Connect " << uri << " failed."; + } +// if (channel_ != nullptr) { +// connected_ = true; +// client_ptr_ = std::make_shared(channel_); +// return Status::OK(); +// } + +// std::string reason = "connect failed!"; +// connected_ = false; +// return Status(StatusCode::NotConnected, reason); } Status diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index e332266acf..572b782769 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -105,7 +105,7 @@ class ClientProxy : public Connection { DropPartition(const PartitionParam& partition_param) override; private: - std::shared_ptr<::grpc::Channel> channel_; + std::shared_ptr channel_; private: std::shared_ptr client_ptr_; From 4c71748bb2b423fc56a9c63c44c2f3acefae371b Mon Sep 17 00:00:00 2001 From: erdustiggen Date: Sat, 16 Nov 2019 20:55:16 +0100 Subject: [PATCH 06/82] Fix typos and grammatical errors --- core/src/wrapper/VecImpl.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index c97900f839..931eef9dbf 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -26,7 +26,7 @@ /* * no parameter check in this layer. - * only responible for index combination + * only responsible for index combination */ namespace milvus { @@ -271,7 +271,7 @@ IVFHybridIndex::LoadQuantizer(const Config& conf) { if (auto new_idx = std::dynamic_pointer_cast(index_)) { return new_idx->LoadQuantizer(conf); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); } } @@ -282,8 +282,8 @@ IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) { if (auto new_idx = std::dynamic_pointer_cast(index_)) { new_idx->SetQuantizer(q); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); - return Status(KNOWHERE_ERROR, "not support"); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + return Status(KNOWHERE_ERROR, "not supported"); } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); @@ -302,8 +302,8 @@ IVFHybridIndex::UnsetQuantizer() { if (auto new_idx = std::dynamic_pointer_cast(index_)) { new_idx->UnsetQuantizer(); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); - return Status(KNOWHERE_ERROR, "not support"); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + return Status(KNOWHERE_ERROR, "not supported"); } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); @@ -322,7 +322,7 @@ IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { if (auto new_idx = std::dynamic_pointer_cast(index_)) { return std::make_shared(new_idx->LoadData(q, conf), type); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); @@ -341,7 +341,7 @@ IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& c auto new_idx = std::make_shared(pair.first, type); return std::make_pair(new_idx, pair.second); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); From d5eb5675b77210752140184df8a5819edfa92e95 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Mon, 18 Nov 2019 09:53:38 +0800 Subject: [PATCH 07/82] fix by reduce knng size --- core/src/index/CMakeLists.txt | 6 +- .../index/knowhere/knowhere/common/Config.h | 14 ++ .../vector_index/helpers/IndexParameter.cpp | 22 ++++ .../vector_index/helpers/IndexParameter.h | 9 ++ core/src/wrapper/ConfAdapter.cpp | 27 ++-- core/src/wrapper/ConfAdapter.h | 3 - core/unittest/wrapper/test_wrapper.cpp | 121 +++++++++++++++++- core/unittest/wrapper/utils.cpp | 5 + core/unittest/wrapper/utils.h | 10 +- 9 files changed, 188 insertions(+), 29 deletions(-) diff --git a/core/src/index/CMakeLists.txt b/core/src/index/CMakeLists.txt index 4b5c1b1de3..53453d53aa 100644 --- a/core/src/index/CMakeLists.txt +++ b/core/src/index/CMakeLists.txt @@ -88,14 +88,14 @@ endif () include(ThirdPartyPackagesCore) if (CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") if (KNOWHERE_GPU_VERSION) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") endif () else () - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") if (KNOWHERE_GPU_VERSION) - set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3 -g") endif () endif () diff --git a/core/src/index/knowhere/knowhere/common/Config.h b/core/src/index/knowhere/knowhere/common/Config.h index 6191ecb771..48c2de8b1b 100644 --- a/core/src/index/knowhere/knowhere/common/Config.h +++ b/core/src/index/knowhere/knowhere/common/Config.h @@ -18,6 +18,8 @@ #pragma once #include +#include +#include "Log.h" namespace knowhere { @@ -50,6 +52,18 @@ struct Cfg { CheckValid() { return true; } + + void + Dump() { + KNOWHERE_LOG_DEBUG << DumpImpl().str(); + } + + virtual std::stringstream + DumpImpl() { + std::stringstream ss; + ss << "dim: " << d << ", metric: " << int(metric_type) << ", gpuid: " << gpu_id << ", k: " << k; + return ss; + } }; using Config = std::shared_ptr; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp index 20f3388174..6e9ee55658 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp @@ -34,4 +34,26 @@ GetMetricType(METRICTYPE& type) { KNOWHERE_THROW_MSG("Metric type is invalid"); } +std::stringstream +IVFCfg::DumpImpl() { + auto ss = Cfg::DumpImpl(); + ss << ", nlist: " << nlist << ", nprobe: " << nprobe; + return ss; +} + +std::stringstream +IVFSQCfg::DumpImpl() { + auto ss = IVFCfg::DumpImpl(); + ss << ", nbits: " << nbits; + return ss; +} + +std::stringstream +NSGCfg::DumpImpl() { + auto ss = IVFCfg::DumpImpl(); + ss << ", knng: " << knng << ", search_length: " << search_length << ", out_degree: " << out_degree + << ", candidate: " << candidate_pool_size; + return ss; +} + } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h index b2854abef8..b931790b04 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -53,6 +53,9 @@ struct IVFCfg : public Cfg { IVFCfg() = default; + std::stringstream + DumpImpl() override; + bool CheckValid() override { return true; @@ -69,6 +72,9 @@ struct IVFSQCfg : public IVFCfg { : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), nbits(nbits) { } + std::stringstream + DumpImpl() override; + IVFSQCfg() = default; bool @@ -119,6 +125,9 @@ struct NSGCfg : public IVFCfg { NSGCfg() = default; + std::stringstream + DumpImpl() override; + bool CheckValid() override { return true; diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 461745f1fd..b121372a26 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -47,7 +47,8 @@ ConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; + conf->k = metaconf.k; MatchBase(conf); return conf; } @@ -65,7 +66,7 @@ IVFConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; MatchBase(conf); return conf; } @@ -114,7 +115,7 @@ IVFSQConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; MatchBase(conf); return conf; @@ -126,7 +127,7 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; if (!(conf->d % 4)) @@ -175,21 +176,17 @@ NSGConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; + conf->k = metaconf.k; - double factor = metaconf.size / TYPICAL_COUNT; auto scale_factor = round(metaconf.dim / 128.0); scale_factor = scale_factor >= 4 ? 4 : scale_factor; - conf->nprobe = conf->nlist > 10000 ? conf->nlist * 0.02 : conf->nlist * 0.1; - conf->knng = (100 + 100 * scale_factor) * factor; - conf->search_length = (40 + 5 * scale_factor) * factor; - conf->out_degree = (50 + 5 * scale_factor) * factor; - conf->candidate_pool_size = (200 + 100 * scale_factor) * factor; + conf->nprobe = int64_t(conf->nlist * 0.01); + conf->knng = 40 + 10 * scale_factor; // the size of knng + conf->search_length = 40 + 5 * scale_factor; + conf->out_degree = 50 + 5 * scale_factor; + conf->candidate_pool_size = 200 + 100 * scale_factor; MatchBase(conf); - - // WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist - // << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d - // << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng; return conf; } diff --git a/core/src/wrapper/ConfAdapter.h b/core/src/wrapper/ConfAdapter.h index 5ec3d52486..85637a4969 100644 --- a/core/src/wrapper/ConfAdapter.h +++ b/core/src/wrapper/ConfAdapter.h @@ -46,9 +46,6 @@ class ConfAdapter { virtual knowhere::Config MatchSearch(const TempMetaConf& metaconf, const IndexType& type); - // virtual void - // Dump(){} - protected: static void MatchBase(knowhere::Config conf); diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 025601a1cd..11a8b8e01a 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -56,10 +56,6 @@ class KnowhereWrapperTest index_ = GetVecIndexFactory(index_type); conf = ParamGenerator::GetInstance().GenBuild(index_type, tempconf); searchconf = ParamGenerator::GetInstance().GenSearchConf(index_type, tempconf); - -// conf->k = k; -// conf->d = dim; -// conf->gpu_id = DEVICEID; } void TearDown() override { @@ -97,6 +93,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest, TEST_P(KnowhereWrapperTest, BASE_TEST) { EXPECT_EQ(index_->GetType(), index_type); + // conf->Dump(); auto elems = nq * k; std::vector res_ids(elems); @@ -191,3 +188,119 @@ TEST(whatever, test_config) { auto pq_conf = std::make_shared(); pq_conf->Match(conf); } + +// #include "knowhere/index/vector_index/IndexIDMAP.h" +// #include "src/wrapper/VecImpl.h" +// #include "src/index/unittest/utils.h" +// The two case below prove NSG is concern with data distribution +// Further work: 1. Use right basedata and pass it by milvus +// a. batch size is 100000 [Pass] +// b. transfer all at once [Pass] +// 2. Use SIFT1M in test and check time cost [] +// TEST_P(KnowhereWrapperTest, nsgwithidmap) { +// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP); +// auto ori_xb = xb; +// auto ori_ids = ids; +// std::vector temp_xb; +// std::vector temp_ids; +// nb = 50000; +// for (int i = 0; i < 20; ++i) { +// GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis); +// assert(xb.size() == nb*dim); +// //#define IDMAP +// #ifdef IDMAP +// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim); +// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1)); +// if (i == 0) { +// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf); +// } else { +// idmap->Add(nb, temp_xb.data(), temp_ids.data()); +// } +// temp_xb.clear(); +// temp_ids.clear(); +// #else +// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim); +// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1)); +// #endif +// } + +// #ifdef IDMAP +// auto idmap_idx = std::dynamic_pointer_cast(idmap); +// auto x = idmap_idx->Count(); +// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf); +// #else +// assert(temp_xb.size() == 1000000*128); +// index_->BuildAll(1000000, temp_xb.data(), ori_ids.data(), conf); +// #endif +// } + +// TEST_P(KnowhereWrapperTest, nsgwithsidmap) { +// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP); +// auto ori_xb = xb; +// std::vector temp_xb; +// std::vector temp_ids; +// nb = 50000; +// for (int i = 0; i < 20; ++i) { +// #define IDMAP +// #ifdef IDMAP +// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1)); +// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1)); +// if (i == 0) { +// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf); +// } else { +// idmap->Add(nb, temp_xb.data(), temp_ids.data()); +// } +// temp_xb.clear(); +// temp_ids.clear(); +// #else +// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1)); +// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1)); +// #endif +// } + +// #ifdef IDMAP +// auto idmap_idx = std::dynamic_pointer_cast(idmap); +// auto x = idmap_idx->Count(); +// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf); +// #else +// index_->BuildAll(1000000, temp_xb.data(), temp_ids.data(), conf); +// #endif + +// // The code use to store raw base data +// FileIOWriter writer("/tmp/newraw"); +// ori_xb.shrink_to_fit(); +// std::cout << "size" << ori_xb.size(); +// writer(static_cast(ori_xb.data()), ori_xb.size()* sizeof(float)); +// std::cout << "Finish!" << std::endl; +// } + +// void load_data(char* filename, float*& data, unsigned& num, +// unsigned& dim) { // load data with sift10K pattern +// std::ifstream in(filename, std::ios::binary); +// if (!in.is_open()) { +// std::cout << "open file error" << std::endl; +// exit(-1); +// } +// in.read((char*)&dim, 4); +// in.seekg(0, std::ios::end); +// std::ios::pos_type ss = in.tellg(); +// size_t fsize = (size_t)ss; +// num = (unsigned)(fsize / (dim + 1) / 4); +// data = new float[(size_t)num * (size_t)dim]; + +// in.seekg(0, std::ios::beg); +// for (size_t i = 0; i < num; i++) { +// in.seekg(4, std::ios::cur); +// in.read((char*)(data + i * dim), dim * 4); +// } +// in.close(); +// } + +// TEST_P(KnowhereWrapperTest, Sift1M) { +// float* data = nullptr; +// unsigned points_num, dim; +// load_data("/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Data/sift/sift_base.fvecs", data, points_num, +// dim); std::cout << points_num << " " << dim << std::endl; + +// index_->BuildAll(points_num, data, ids.data(), conf); +// } \ No newline at end of file diff --git a/core/unittest/wrapper/utils.cpp b/core/unittest/wrapper/utils.cpp index 00a1db2c88..96b9e643f5 100644 --- a/core/unittest/wrapper/utils.cpp +++ b/core/unittest/wrapper/utils.cpp @@ -117,6 +117,11 @@ void DataGenBase::GenData(const int& dim, const int& nb, const int& nq, std::vector& xb, std::vector& xq, std::vector& ids, const int& k, std::vector& gt_ids, std::vector& gt_dis) { + xb.clear(); + xq.clear(); + ids.clear(); + gt_ids.clear(); + gt_dis.clear(); xb.resize(nb * dim); xq.resize(nq * dim); ids.resize(nb); diff --git a/core/unittest/wrapper/utils.h b/core/unittest/wrapper/utils.h index 1eaa7ae4ec..cc1926302d 100644 --- a/core/unittest/wrapper/utils.h +++ b/core/unittest/wrapper/utils.h @@ -25,7 +25,6 @@ #include #include - #include "wrapper/VecIndex.h" #include "wrapper/utils.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" @@ -90,17 +89,20 @@ class ParamGenerator { return instance; } - knowhere::Config GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { + knowhere::Config + GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->MatchSearch(conf, type); } - knowhere::Config GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { + knowhere::Config + GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->Match(conf); } - knowhere::Config Gen(const milvus::engine::IndexType& type) { + knowhere::Config + Gen(const milvus::engine::IndexType& type) { switch (type) { case milvus::engine::IndexType::FAISS_IDMAP: { auto tempconf = std::make_shared(); From 82cb637ca8760828e3095783ca0b600d165c90d1 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Mon, 18 Nov 2019 10:50:20 +0800 Subject: [PATCH 08/82] code style format --- core/src/wrapper/ConfAdapter.cpp | 2 +- core/unittest/wrapper/test_wrapper.cpp | 2 +- core/unittest/wrapper/utils.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index b121372a26..0ab38d3394 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -182,7 +182,7 @@ NSGConfAdapter::Match(const TempMetaConf& metaconf) { auto scale_factor = round(metaconf.dim / 128.0); scale_factor = scale_factor >= 4 ? 4 : scale_factor; conf->nprobe = int64_t(conf->nlist * 0.01); - conf->knng = 40 + 10 * scale_factor; // the size of knng + conf->knng = 40 + 10 * scale_factor; // the size of knng conf->search_length = 40 + 5 * scale_factor; conf->out_degree = 50 + 5 * scale_factor; conf->candidate_pool_size = 200 + 100 * scale_factor; diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 11a8b8e01a..a07fafc7b8 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -303,4 +303,4 @@ TEST(whatever, test_config) { // dim); std::cout << points_num << " " << dim << std::endl; // index_->BuildAll(points_num, data, ids.data(), conf); -// } \ No newline at end of file +// } diff --git a/core/unittest/wrapper/utils.h b/core/unittest/wrapper/utils.h index cc1926302d..05dc92f2d6 100644 --- a/core/unittest/wrapper/utils.h +++ b/core/unittest/wrapper/utils.h @@ -89,13 +89,13 @@ class ParamGenerator { return instance; } - knowhere::Config + knowhere::Config GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->MatchSearch(conf, type); } - knowhere::Config + knowhere::Config GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->Match(conf); From 513ad3b8424d27f886fd6162e8fea6f611be5293 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Mon, 18 Nov 2019 11:38:48 +0800 Subject: [PATCH 09/82] Support build index with multiple gpu --- CHANGELOG.md | 1 + core/conf/server_cpu_config.template | 3 +- core/conf/server_gpu_config.template | 3 +- core/src/db/engine/ExecutionEngineImpl.cpp | 11 +- core/src/scheduler/JobMgr.cpp | 2 +- core/src/scheduler/SchedInst.cpp | 33 ++++-- core/src/scheduler/SchedInst.h | 5 + .../scheduler/action/PushTaskToNeighbour.cpp | 102 ++++++------------ .../scheduler/optimizer/BuildIndexPass.cpp | 46 ++++++++ core/src/scheduler/optimizer/BuildIndexPass.h | 51 +++++++++ core/src/scheduler/optimizer/OnlyGPUPass.cpp | 3 +- core/src/server/Config.cpp | 72 ++++++++----- core/src/server/Config.h | 7 +- core/src/wrapper/KnowhereResource.cpp | 8 +- core/unittest/server/test_config.cpp | 3 +- 15 files changed, 233 insertions(+), 117 deletions(-) create mode 100644 core/src/scheduler/optimizer/BuildIndexPass.cpp create mode 100644 core/src/scheduler/optimizer/BuildIndexPass.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 745310faa7..42d9291573 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#12 - Pure CPU version for Milvus - \#77 - Support table partition - \#226 - Experimental shards middleware for Milvus +- \#346 - Support build index with multiple gpu ## Improvement - \#275 - Rename C++ SDK IndexType diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 6c95126390..bcfb5fa7ce 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -40,4 +40,5 @@ engine_config: resource_config: search_resources: # define the device used for search computation - cpu - index_build_device: cpu # CPU used for building index + index_build_device: # CPU used for building index + - cpu diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 154db5d134..a347a9e5bd 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -42,4 +42,5 @@ resource_config: search_resources: # define the devices used for search computation, must be in format: cpu or gpux - cpu - gpu0 - index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu or gpux + index_build_device: # CPU / GPU used for building index, must be in format: cpu or gpux + - gpu0 diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 19c699bda7..dd80ec796b 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -570,12 +570,19 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); - Status s = config.GetResourceConfigIndexBuildDevice(gpu_num_); + std::vector gpu_ids; + Status s = config.GetResourceConfigIndexBuildDevice(gpu_ids); if (!s.ok()) { return s; } + for (auto id : gpu_ids) { + if (gpu_num_ == id) { + return Status::OK(); + } + } - return Status::OK(); + std::string msg = "Invalid gpu_num"; + return Status(SERVER_INVALID_ARGUMENT, msg); } } // namespace engine diff --git a/core/src/scheduler/JobMgr.cpp b/core/src/scheduler/JobMgr.cpp index 794f6a0f37..8dd095a3fa 100644 --- a/core/src/scheduler/JobMgr.cpp +++ b/core/src/scheduler/JobMgr.cpp @@ -104,7 +104,7 @@ JobMgr::build_task(const JobPtr& job) { void JobMgr::calculate_path(const TaskPtr& task) { - if (task->type_ != TaskType::SearchTask) { + if (task->type_ != TaskType::SearchTask && task->type_ != TaskType::BuildIndexTask) { return; } diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 61e0c09759..407c3a44d1 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -54,8 +54,8 @@ load_simple_config() { // get resources auto gpu_ids = get_gpu_pool(); - int32_t build_gpu_id; - config.GetResourceConfigIndexBuildDevice(build_gpu_id); + std::vector build_gpu_ids; + config.GetResourceConfigIndexBuildDevice(build_gpu_ids); // create and connect ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false)); @@ -65,19 +65,30 @@ load_simple_config() { ResMgrInst::GetInstance()->Connect("disk", "cpu", io); auto pcie = Connection("pcie", 12000); - bool find_build_gpu_id = false; - for (auto& gpu_id : gpu_ids) { - ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); - if (build_gpu_id == gpu_id) { - find_build_gpu_id = true; + + std::vector not_find_build_ids; + for (auto& build_id : build_gpu_ids) { + bool find_gpu_id = false; + for (auto& gpu_id : gpu_ids) { + if (gpu_id == build_id) { + find_gpu_id = true; + break; + } + } + if (not find_gpu_id) { + not_find_build_ids.emplace_back(build_id); } } - if (not find_build_gpu_id) { + for (auto& gpu_id : gpu_ids) { + ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); + } + + for (auto& not_find_id : not_find_build_ids) { ResMgrInst::GetInstance()->Add( - ResourceFactory::Create(std::to_string(build_gpu_id), "GPU", build_gpu_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(build_gpu_id), pcie); + ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie); } } diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index a3048069f9..4fd22f1fab 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -21,6 +21,7 @@ #include "JobMgr.h" #include "ResourceMgr.h" #include "Scheduler.h" +#include "optimizer/BuildIndexPass.h" #include "optimizer/HybridPass.h" #include "optimizer/LargeSQ8HPass.h" #include "optimizer/OnlyCPUPass.h" @@ -107,11 +108,15 @@ class OptimizerInst { } } + std::vector build_resources; + config.GetResourceConfigIndexBuildDevice(build_resources); + std::vector pass_list; pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared(has_cpu)); + pass_list.push_back(std::make_shared(build_resources)); instance = std::make_shared(pass_list); } } diff --git a/core/src/scheduler/action/PushTaskToNeighbour.cpp b/core/src/scheduler/action/PushTaskToNeighbour.cpp index b8a4a1164b..9aed678937 100644 --- a/core/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/core/src/scheduler/action/PushTaskToNeighbour.cpp @@ -138,73 +138,41 @@ Action::SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, Resou std::shared_ptr event) { auto task_item = event->task_table_item_; auto task = event->task_table_item_->task; - if (resource->type() == ResourceType::DISK) { - // step 1: calculate shortest path per resource, from disk to compute resource - auto compute_resources = res_mgr->GetComputeResources(); - std::vector> paths; - std::vector transport_costs; - for (auto& res : compute_resources) { - std::vector path; - uint64_t transport_cost = ShortestPath(resource, res, res_mgr, path); - transport_costs.push_back(transport_cost); - paths.emplace_back(path); - } - // if (task->job_.lock()->type() == JobType::SEARCH) { - // auto label = task->label(); - // auto spec_label = std::static_pointer_cast(label); - // if (spec_label->resource().lock()->type() == ResourceType::CPU) { - // std::vector spec_path; - // spec_path.push_back(spec_label->resource().lock()->name()); - // spec_path.push_back(resource->name()); - // task->path() = Path(spec_path, spec_path.size() - 1); - // } else { - // // step 2: select min cost, cost(resource) = avg_cost * task_to_do + transport_cost - // uint64_t min_cost = std::numeric_limits::max(); - // uint64_t min_cost_idx = 0; - // for (uint64_t i = 0; i < compute_resources.size(); ++i) { - // if (compute_resources[i]->TotalTasks() == 0) { - // min_cost_idx = i; - // break; - // } - // uint64_t cost = compute_resources[i]->TaskAvgCost() * - // compute_resources[i]->NumOfTaskToExec() + - // transport_costs[i]; - // if (min_cost > cost) { - // min_cost = cost; - // min_cost_idx = i; - // } - // } - // - // // step 3: set path in task - // Path task_path(paths[min_cost_idx], paths[min_cost_idx].size() - 1); - // task->path() = task_path; - // } - // - // } else - if (task->job_.lock()->type() == JobType::BUILD) { - // step2: Read device id in config - // get build index gpu resource - server::Config& config = server::Config::GetInstance(); - int32_t build_index_gpu; - Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); - - bool find_gpu_res = false; - if (res_mgr->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { - for (uint64_t i = 0; i < compute_resources.size(); ++i) { - if (compute_resources[i]->name() == - res_mgr->GetResource(ResourceType::GPU, build_index_gpu)->name()) { - find_gpu_res = true; - Path task_path(paths[i], paths[i].size() - 1); - task->path() = task_path; - break; - } - } - } - if (not find_gpu_res) { - task->path() = Path(paths[0], paths[0].size() - 1); - } - } - } + // if (resource->type() == ResourceType::DISK) { + // // step 1: calculate shortest path per resource, from disk to compute resource + // auto compute_resources = res_mgr->GetComputeResources(); + // std::vector> paths; + // std::vector transport_costs; + // for (auto& res : compute_resources) { + // std::vector path; + // uint64_t transport_cost = ShortestPath(resource, res, res_mgr, path); + // transport_costs.push_back(transport_cost); + // paths.emplace_back(path); + // } + // if (task->job_.lock()->type() == JobType::BUILD) { + // // step2: Read device id in config + // // get build index gpu resource + // server::Config& config = server::Config::GetInstance(); + // int32_t build_index_gpu; + // Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); + // + // bool find_gpu_res = false; + // if (res_mgr->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { + // for (uint64_t i = 0; i < compute_resources.size(); ++i) { + // if (compute_resources[i]->name() == + // res_mgr->GetResource(ResourceType::GPU, build_index_gpu)->name()) { + // find_gpu_res = true; + // Path task_path(paths[i], paths[i].size() - 1); + // task->path() = task_path; + // break; + // } + // } + // } + // if (not find_gpu_res) { + // task->path() = Path(paths[0], paths[0].size() - 1); + // } + // } + // } if (resource->name() == task->path().Last()) { resource->WakeupExecutor(); diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp new file mode 100644 index 0000000000..725b2509c3 --- /dev/null +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/BuildIndexPass.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/tasklabel/SpecResLabel.h" + +namespace milvus { +namespace scheduler { + +BuildIndexPass::BuildIndexPass(std::vector& build_gpu_ids) : build_gpu_ids_(build_gpu_ids) { +} + +bool +BuildIndexPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::BuildIndexTask) + return false; + + if (build_gpu_ids_.empty()) + return false; + + ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_gpu_ids_[specified_gpu_id_]); + auto label = std::make_shared(std::weak_ptr(res_ptr)); + task->label() = label; + + specified_gpu_id_ = (specified_gpu_id_ + 1) % build_gpu_ids_.size(); + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/BuildIndexPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h new file mode 100644 index 0000000000..a70844bfa9 --- /dev/null +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class BuildIndexPass : public Pass { + public: + explicit BuildIndexPass(std::vector& build_gpu_id); + + public: + bool + Run(const TaskPtr& task) override; + + private: + uint64_t specified_gpu_id_ = 0; + std::vector build_gpu_ids_; +}; + +using BuildIndexPassPtr = std::shared_ptr; + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.cpp b/core/src/scheduler/optimizer/OnlyGPUPass.cpp index 3fcda0e8a3..e867e45159 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.cpp +++ b/core/src/scheduler/optimizer/OnlyGPUPass.cpp @@ -41,12 +41,11 @@ OnlyGPUPass::Run(const TaskPtr& task) { auto gpu_id = get_gpu_pool(); if (gpu_id.empty()) return false; - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpu_id[specified_gpu_id_]); auto label = std::make_shared(std::weak_ptr(res_ptr)); task->label() = label; - specified_gpu_id_ = specified_gpu_id_++ % gpu_id.size(); + specified_gpu_id_ = (specified_gpu_id_ + 1) % gpu_id.size(); return true; } diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index f130e73a85..a2a1354dd4 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -215,8 +215,8 @@ Config::ValidateConfig() { return s; } - int32_t resource_index_build_device; - s = GetResourceConfigIndexBuildDevice(resource_index_build_device); + std::vector index_build_devices; + s = GetResourceConfigIndexBuildDevice(index_build_devices); if (!s.ok()) { return s; } @@ -599,22 +599,24 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } else { uint64_t gpu_cache_capacity = std::stoi(value) * GB; - int device_id; - Status s = GetResourceConfigIndexBuildDevice(device_id); + std::vector device_ids; + Status s = GetResourceConfigIndexBuildDevice(device_ids); if (!s.ok()) { return s; } size_t gpu_memory; - if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { - std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); - return Status(SERVER_UNEXPECTED_ERROR, msg); - } else if (gpu_cache_capacity >= gpu_memory) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { - std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + for (auto& device_id : device_ids) { + if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { + std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); + return Status(SERVER_UNEXPECTED_ERROR, msg); + } else if (gpu_cache_capacity >= gpu_memory) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { + std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + } } } return Status::OK(); @@ -745,11 +747,21 @@ Config::CheckResourceConfigSearchResources(const std::vector& value } Status -Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { - auto status = CheckResource(value); - if (!status.ok()) { - return Status(SERVER_INVALID_ARGUMENT, status.message()); +Config::CheckResourceConfigIndexBuildDevice(const std::vector& value) { + if (value.empty()) { + std::string msg = + "Invalid index build resource. " + "Possible reason: resource_config.index_build_device is empty."; + return Status(SERVER_INVALID_ARGUMENT, msg); } + + for (auto& resource : value) { + auto status = CheckResource(resource); + if (!status.ok()) { + return Status(SERVER_INVALID_ARGUMENT, status.message()); + } + } + return Status::OK(); } @@ -1036,18 +1048,25 @@ Config::GetResourceConfigSearchResources(std::vector& value) { } Status -Config::GetResourceConfigIndexBuildDevice(int32_t& value) { +Config::GetResourceConfigIndexBuildDevice(std::vector& value) { std::string str = - GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); - Status s = CheckResourceConfigIndexBuildDevice(str); + GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DELIMITER, + CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + std::vector resources; + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_INDEX_BUILD_DELIMITER, resources); + + Status s = CheckResourceConfigIndexBuildDevice(resources); if (!s.ok()) { return s; } - if (str == "cpu") { - value = CPU_DEVICE_ID; - } else { - value = std::stoi(str.substr(3)); + for (auto res : resources) { + if (res == "cpu") { + value.emplace_back(CPU_DEVICE_ID); + break; + } + int64_t device_id = std::stoi(str.substr(3)); + value.emplace_back(device_id); } return Status::OK(); @@ -1318,7 +1337,10 @@ Config::SetResourceConfigSearchResources(const std::string& value) { Status Config::SetResourceConfigIndexBuildDevice(const std::string& value) { - Status s = CheckResourceConfigIndexBuildDevice(value); + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_INDEX_BUILD_DELIMITER, res_vec); + + Status s = CheckResourceConfigIndexBuildDevice(res_vec); if (!s.ok()) { return s; } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 3ab0cd8053..08b3accdd6 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -101,10 +101,11 @@ static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0"; #endif static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_DELIMITER = ","; #ifdef MILVUS_CPU_VERSION static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu"; #else -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu,gpu0"; #endif const int32_t CPU_DEVICE_ID = -1; @@ -190,7 +191,7 @@ class Config { Status CheckResourceConfigSearchResources(const std::vector& value); Status - CheckResourceConfigIndexBuildDevice(const std::string& value); + CheckResourceConfigIndexBuildDevice(const std::vector& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -259,7 +260,7 @@ class Config { Status GetResourceConfigSearchResources(std::vector& value); Status - GetResourceConfigIndexBuildDevice(int32_t& value); + GetResourceConfigIndexBuildDevice(std::vector& value); public: /* server config */ diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index 8ed19232e9..ccfbcbb6cf 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -48,12 +48,14 @@ KnowhereResource::Initialize() { // get build index gpu resource server::Config& config = server::Config::GetInstance(); - int32_t build_index_gpu; - s = config.GetResourceConfigIndexBuildDevice(build_index_gpu); + std::vector build_index_gpus; + s = config.GetResourceConfigIndexBuildDevice(build_index_gpus); if (!s.ok()) return s; - gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); + for (auto gpu_id : build_index_gpus) { + gpu_resources.insert(std::make_pair(gpu_id, GpuResourceSetting())); + } // get search gpu resource std::vector pool; diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 637273732d..7f4376bb67 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -292,7 +292,8 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.SetResourceConfigIndexBuildDevice("gpu" + std::to_string(resource_index_build_device)); #endif ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigIndexBuildDevice(int32_val); + std::vector device_ids; + s = config.GetResourceConfigIndexBuildDevice(device_ids); ASSERT_TRUE(s.ok()); ASSERT_TRUE(int32_val == resource_index_build_device); } From ead945870cd8d742f870862e6c60031d22841c0a Mon Sep 17 00:00:00 2001 From: zhenwu Date: Mon, 18 Nov 2019 15:49:37 +0800 Subject: [PATCH 10/82] Update nightly-build config --- ci/jenkins/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index f0562ada64..8d3953b112 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -2,7 +2,7 @@ String cron_timezone = "TZ=Asia/Shanghai" String cron_string = BRANCH_NAME == "master" ? "H 0 * * * " : "" -cron_string = BRANCH_NAME == "0.5.1" ? "H 1 * * * " : cron_string +cron_string = BRANCH_NAME == "0.6.0" ? "H 1 * * * " : cron_string pipeline { agent none From 661a60374e2f9d10cd73b5a676c01cf7bbe181e6 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 18 Nov 2019 15:58:50 +0800 Subject: [PATCH 11/82] dont modify grpc generated code --- core/src/grpc/gen-milvus/milvus.pb.h | 57 ++++++------------- core/src/sdk/grpc/ClientProxy.cpp | 15 ++++- .../grpc_impl/request/SearchRequest.cpp | 2 +- 3 files changed, 29 insertions(+), 45 deletions(-) diff --git a/core/src/grpc/gen-milvus/milvus.pb.h b/core/src/grpc/gen-milvus/milvus.pb.h index 536365b23b..53ed2db22e 100644 --- a/core/src/grpc/gen-milvus/milvus.pb.h +++ b/core/src/grpc/gen-milvus/milvus.pb.h @@ -1321,8 +1321,7 @@ class RowRecord : void clear_vector_data(); float vector_data(int index) const; void set_vector_data(int index, float value); -// void add_vector_data(float value); - void add_vector_data(std::vector::const_iterator begin, std::vector::const_iterator end); + void add_vector_data(float value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& vector_data() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* @@ -1474,9 +1473,7 @@ class InsertParam : void clear_row_id_array(); ::PROTOBUF_NAMESPACE_ID::int64 row_id_array(int index) const; void set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::int64 value); -// void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value); - void add_row_id_array(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end); + void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& row_id_array() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* @@ -2139,9 +2136,7 @@ class TopKQueryResult : void clear_ids(); ::PROTOBUF_NAMESPACE_ID::int64 ids(int index) const; void set_ids(int index, ::PROTOBUF_NAMESPACE_ID::int64 value); -// void add_ids(::PROTOBUF_NAMESPACE_ID::int64 value); - void add_ids(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end); + void add_ids(::PROTOBUF_NAMESPACE_ID::int64 value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& ids() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* @@ -2152,8 +2147,7 @@ class TopKQueryResult : void clear_distances(); float distances(int index) const; void set_distances(int index, float value); -// void add_distances(float value); - void add_distances(std::vector::const_iterator begin, std::vector::const_iterator end); + void add_distances(float value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& distances() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* @@ -3928,14 +3922,9 @@ inline void RowRecord::set_vector_data(int index, float value) { vector_data_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.RowRecord.vector_data) } -//inline void RowRecord::add_vector_data(float value) { -// vector_data_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) -//} -inline void RowRecord::add_vector_data(std::vector::const_iterator begin, - std::vector::const_iterator end) { - vector_data_.Add(begin, end); - // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) +inline void RowRecord::add_vector_data(float value) { + vector_data_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& RowRecord::vector_data() const { @@ -4048,14 +4037,9 @@ inline void InsertParam::set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::in row_id_array_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.InsertParam.row_id_array) } -//inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) { -// row_id_array_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) -//} -inline void InsertParam::add_row_id_array(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end) { - row_id_array_.Add(begin, end); - // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) +inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) { + row_id_array_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& InsertParam::row_id_array() const { @@ -4604,14 +4588,9 @@ inline void TopKQueryResult::set_ids(int index, ::PROTOBUF_NAMESPACE_ID::int64 v ids_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.TopKQueryResult.ids) } -//inline void TopKQueryResult::add_ids(::PROTOBUF_NAMESPACE_ID::int64 value) { -// ids_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) -//} -inline void TopKQueryResult::add_ids(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end) { - ids_.Add(begin,end); - // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) +inline void TopKQueryResult::add_ids(::PROTOBUF_NAMESPACE_ID::int64 value) { + ids_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& TopKQueryResult::ids() const { @@ -4639,13 +4618,9 @@ inline void TopKQueryResult::set_distances(int index, float value) { distances_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.TopKQueryResult.distances) } -//inline void TopKQueryResult::add_distances(float value) { -// distances_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) -//} -inline void TopKQueryResult::add_distances(std::vector::const_iterator begin, std::vector::const_iterator end) { - distances_.Add(begin, end); - // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) +inline void TopKQueryResult::add_distances(float value) { + distances_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& TopKQueryResult::distances() const { diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 131d0bdc99..4a9c319b4d 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -32,6 +32,13 @@ UriCheck(const std::string& uri) { return (index != std::string::npos); } +void +CopyRowRecord(::milvus::grpc::RowRecord* target, const RowRecord& src) { + auto vector_data = target->mutable_vector_data(); + vector_data->Resize(static_cast(src.data.size()), 0.0); + memcpy(vector_data->mutable_data(), src.data.data(), src.data.size() * sizeof(float)); +} + Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; @@ -189,14 +196,16 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_ for (auto& record : record_array) { ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); - grpc_record->add_vector_data(record.data.begin(), record.data.end()); + CopyRowRecord(grpc_record, record); } // Single thread ::milvus::grpc::VectorIds vector_ids; if (!id_array.empty()) { /* set user's ids */ - insert_param.add_row_id_array(id_array.begin(), id_array.end()); + auto row_ids = insert_param.mutable_row_id_array(); + row_ids->Resize(static_cast(id_array.size()), -1); + memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); client_ptr_->Insert(vector_ids, insert_param, status); } else { client_ptr_->Insert(vector_ids, insert_param, status); @@ -226,7 +235,7 @@ ClientProxy::Search(const std::string& table_name, const std::vectoradd_vector_data(record.data.begin(), record.data.end()); + CopyRowRecord(row_record, record); } // step 2: convert range array diff --git a/core/src/server/grpc_impl/request/SearchRequest.cpp b/core/src/server/grpc_impl/request/SearchRequest.cpp index 955576360a..28f4ff723e 100644 --- a/core/src/server/grpc_impl/request/SearchRequest.cpp +++ b/core/src/server/grpc_impl/request/SearchRequest.cpp @@ -167,7 +167,7 @@ SearchRequest::OnExecute() { // step 7: construct result array topk_result_->set_row_num(record_count); - topk_result_->mutable_ids()->Resize(static_cast(result_ids.size()), 0); + topk_result_->mutable_ids()->Resize(static_cast(result_ids.size()), -1); memcpy(topk_result_->mutable_ids()->mutable_data(), result_ids.data(), result_ids.size() * sizeof(int64_t)); topk_result_->mutable_distances()->Resize(static_cast(result_distances.size()), 0.0); memcpy(topk_result_->mutable_distances()->mutable_data(), result_distances.data(), From b36905055bb18d87e12695ed8335b7022f6f85a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=96=E9=BE=99?= Date: Mon, 18 Nov 2019 17:25:38 +0800 Subject: [PATCH 12/82] update add configurable DB settings --- shards/mishards/__init__.py | 4 +++- shards/mishards/db_base.py | 7 ++----- shards/mishards/settings.py | 11 +++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/shards/mishards/__init__.py b/shards/mishards/__init__.py index a3c55c4ae3..55594220d3 100644 --- a/shards/mishards/__init__.py +++ b/shards/mishards/__init__.py @@ -11,7 +11,9 @@ grpc_server = Server() def create_app(testing_config=None): config = testing_config if testing_config else settings.DefaultConfig - db.init_db(uri=config.SQLALCHEMY_DATABASE_URI, echo=config.SQL_ECHO) + db.init_db(uri=config.SQLALCHEMY_DATABASE_URI, echo=config.SQL_ECHO, pool_size=config.SQL_POOL_SIZE, + pool_recycle=config.SQL_POOL_RECYCLE, pool_timeout=config.SQL_POOL_TIMEOUT, + pool_pre_ping=config.SQL_POOL_PRE_PING, max_overflow=config.SQL_MAX_OVERFLOW) from mishards.connections import ConnectionMgr connect_mgr = ConnectionMgr() diff --git a/shards/mishards/db_base.py b/shards/mishards/db_base.py index 5f2eee9ba1..e55c330352 100644 --- a/shards/mishards/db_base.py +++ b/shards/mishards/db_base.py @@ -23,15 +23,12 @@ class DB: uri and self.init_db(uri, echo) self.session_factory = scoped_session(sessionmaker(class_=LocalSession, db=self)) - def init_db(self, uri, echo=False): + def init_db(self, uri, echo=False, pool_size=100, pool_recycle=5, pool_timeout=30, pool_pre_ping=True, max_overflow=0): url = make_url(uri) if url.get_backend_name() == 'sqlite': self.engine = create_engine(url) else: - self.engine = create_engine(uri, pool_size=100, pool_recycle=5, pool_timeout=30, - pool_pre_ping=True, - echo=echo, - max_overflow=0) + self.engine = create_engine(uri, pool_size, pool_recycle, pool_timeout, pool_pre_ping, echo, max_overflow) self.uri = uri self.url = url diff --git a/shards/mishards/settings.py b/shards/mishards/settings.py index 8d7361dddc..832f1639ea 100644 --- a/shards/mishards/settings.py +++ b/shards/mishards/settings.py @@ -50,10 +50,16 @@ class TracingConfig: } } + max_overflow=0 class DefaultConfig: SQLALCHEMY_DATABASE_URI = env.str('SQLALCHEMY_DATABASE_URI') SQL_ECHO = env.bool('SQL_ECHO', False) + SQL_POOL_SIZE = env.int('pool_size', 100) + SQL_POOL_RECYCLE = env.int('pool_recycle', 5) + SQL_POOL_TIMEOUT = env.int('pool_timeout', 30) + SQL_POOL_PRE_PING = env.bool('pool_pre_ping', True) + SQL_MAX_OVERFLOW = env.int('max_overflow', 0) TRACER_PLUGIN_PATH = env.str('TRACER_PLUGIN_PATH', '') TRACER_CLASS_NAME = env.str('TRACER_CLASS_NAME', '') ROUTER_PLUGIN_PATH = env.str('ROUTER_PLUGIN_PATH', '') @@ -65,5 +71,10 @@ class DefaultConfig: class TestingConfig(DefaultConfig): SQLALCHEMY_DATABASE_URI = env.str('SQLALCHEMY_DATABASE_TEST_URI', '') SQL_ECHO = env.bool('SQL_TEST_ECHO', False) + SQL_POOL_SIZE = env.int('pool_size', 100) + SQL_POOL_RECYCLE = env.int('pool_recycle', 5) + SQL_POOL_TIMEOUT = env.int('pool_timeout', 30) + SQL_POOL_PRE_PING = env.bool('pool_pre_ping', True) + SQL_MAX_OVERFLOW = env.int('max_overflow', 0) TRACER_CLASS_NAME = env.str('TRACER_CLASS_TEST_NAME', '') ROUTER_CLASS_NAME = env.str('ROUTER_CLASS_TEST_NAME', 'FileBasedHashRingRouter') From a5f4484f9d760385a3f972c84eb4377c670fd9ec Mon Sep 17 00:00:00 2001 From: "G.Y Feng" <40229765+GuanyunFeng@users.noreply.github.com> Date: Mon, 18 Nov 2019 18:31:12 +0800 Subject: [PATCH 13/82] Update README_CN.md Add a link to C++ SDK in README_CN #383 --- README_CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_CN.md b/README_CN.md index 5dad64af9b..8f4d4b05aa 100644 --- a/README_CN.md +++ b/README_CN.md @@ -20,7 +20,7 @@ Milvus 是一款开源的、针对海量特征向量的相似性搜索引擎。基于异构众核计算框架设计，成本更低，性能更好。在有限的计算资源下，十亿向量搜索仅毫秒响应。 -Milvus 提供稳定的 Python、Java 以及 C++ 的 API 接口。 +Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java) 以及 [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) 的 API 接口。 通过 [版本发布说明](https://milvus.io/docs/zh-CN/release/v0.5.0/) 获取最新发行版本的 Milvus。 From 039f23e08205831017022d2dcab560d5f279431c Mon Sep 17 00:00:00 2001 From: "G.Y Feng" <40229765+GuanyunFeng@users.noreply.github.com> Date: Mon, 18 Nov 2019 18:32:32 +0800 Subject: [PATCH 14/82] Update README.md Add a link to C++ SDK in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 311d0d6843..4a1151dbf8 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Milvus is the world's fastest similarity search engine for massive-scale feature For more detailed introduction of Milvus and its architecture, see [Milvus overview](https://www.milvus.io/docs/en/aboutmilvus/overview/). -Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and C++ APIs. +Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) APIs. Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://www.milvus.io/docs/en/release/v0.5.0/). From 7e67c76e2c5c9a5791de18e9d5ed131f03ef1681 Mon Sep 17 00:00:00 2001 From: "G.Y Feng" <40229765+GuanyunFeng@users.noreply.github.com> Date: Mon, 18 Nov 2019 18:34:31 +0800 Subject: [PATCH 15/82] Update ubuntu_build_deps.sh solve issue #387, "failed to create symbolic link '/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so': File exists" --- core/ubuntu_build_deps.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/ubuntu_build_deps.sh b/core/ubuntu_build_deps.sh index 911046aa1f..6f3fabc084 100755 --- a/core/ubuntu_build_deps.sh +++ b/core/ubuntu_build_deps.sh @@ -9,4 +9,6 @@ sudo apt-get -y update && sudo apt-get -y install intel-mkl-gnu-2019.5-281 intel sudo apt-get install -y gfortran libmysqlclient-dev mysql-client libcurl4-openssl-dev libboost-system-dev \ libboost-filesystem-dev libboost-serialization-dev libboost-regex-dev -sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so +if [ ! -f "/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so" ]; then + sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so +fi From fddfd1eb03c067234e6127730da14677c03c7554 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Mon, 18 Nov 2019 18:47:00 +0800 Subject: [PATCH 16/82] #346 update config APIs to support build index with multiple GPUs --- core/conf/server_cpu_config.template | 6 +- core/conf/server_gpu_config.template | 3 +- core/src/server/Config.cpp | 84 +++++++++++++++------------- core/src/server/Config.h | 16 +++--- core/src/utils/ValidationUtil.cpp | 4 +- core/src/utils/ValidationUtil.h | 4 +- core/unittest/server/test_config.cpp | 33 ++++++----- 7 files changed, 81 insertions(+), 69 deletions(-) diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 6c95126390..bc8fc3bb3d 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -27,7 +27,6 @@ metric_config: port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] cache_config: - cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] cache_insert_data: false # whether to load inserted data into cache, must be a boolean @@ -38,6 +37,7 @@ engine_config: gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only resource_config: - search_resources: # define the device used for search computation + search_resources: # define the devices used for search computation, must be in format: cpu or gpux + - cpu + index_build_resources: # define the devices used for index building, must be in format: cpu or gpux - cpu - index_build_device: cpu # CPU used for building index diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 154db5d134..c54ed408df 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -42,4 +42,5 @@ resource_config: search_resources: # define the devices used for search computation, must be in format: cpu or gpux - cpu - gpu0 - index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu or gpux + index_build_resources: # define the devices used for index building, must be in format: cpu or gpux + - gpu0 \ No newline at end of file diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index f130e73a85..5672ab52aa 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -215,8 +215,8 @@ Config::ValidateConfig() { return s; } - int32_t resource_index_build_device; - s = GetResourceConfigIndexBuildDevice(resource_index_build_device); + std::vector index_build_resources; + s = GetResourceConfigIndexBuildResources(index_build_resources); if (!s.ok()) { return s; } @@ -351,7 +351,7 @@ Config::ResetDefaultConfig() { return s; } - s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + s = SetResourceConfigIndexBuildResources(CONFIG_RESOURCE_INDEX_BUILD_RESOURCES_DEFAULT); if (!s.ok()) { return s; } @@ -599,22 +599,28 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } else { uint64_t gpu_cache_capacity = std::stoi(value) * GB; - int device_id; - Status s = GetResourceConfigIndexBuildDevice(device_id); + std::vector resources; + Status s = GetResourceConfigIndexBuildResources(resources); if (!s.ok()) { return s; } size_t gpu_memory; - if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { - std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); - return Status(SERVER_UNEXPECTED_ERROR, msg); - } else if (gpu_cache_capacity >= gpu_memory) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { - std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + for (auto& resource : resources) { + if (resource == "cpu") { + continue; + } + int32_t device_id = std::stoi(resource.substr(3)); + if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { + std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); + return Status(SERVER_UNEXPECTED_ERROR, msg); + } else if (gpu_cache_capacity >= gpu_memory) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else if (gpu_cache_capacity > (double) gpu_memory * 0.9) { + std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + } } } return Status::OK(); @@ -745,10 +751,18 @@ Config::CheckResourceConfigSearchResources(const std::vector& value } Status -Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { - auto status = CheckResource(value); - if (!status.ok()) { - return Status(SERVER_INVALID_ARGUMENT, status.message()); +Config::CheckResourceConfigIndexBuildResources(const std::vector& value) { + if (value.empty()) { + std::string msg = + "Invalid build index resource. " + "Possible reason: resource_config.build_index_resources is empty."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + for (auto& resource : value) { + auto status = CheckResource(resource); + if (!status.ok()) { + return Status(SERVER_INVALID_ARGUMENT, status.message()); + } } return Status::OK(); } @@ -1030,27 +1044,18 @@ Status Config::GetResourceConfigSearchResources(std::vector& value) { std::string str = GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, - CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); - server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value); + CONFIG_RESOURCE_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_RESOURCES_DELIMITER, value); return CheckResourceConfigSearchResources(value); } Status -Config::GetResourceConfigIndexBuildDevice(int32_t& value) { +Config::GetResourceConfigIndexBuildResources(std::vector& value) { std::string str = - GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); - Status s = CheckResourceConfigIndexBuildDevice(str); - if (!s.ok()) { - return s; - } - - if (str == "cpu") { - value = CPU_DEVICE_ID; - } else { - value = std::stoi(str.substr(3)); - } - - return Status::OK(); + GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_RESOURCES, + CONFIG_RESOURCE_RESOURCES_DELIMITER, CONFIG_RESOURCE_INDEX_BUILD_RESOURCES_DEFAULT); + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_RESOURCES_DELIMITER, value); + return CheckResourceConfigIndexBuildResources(value); } /////////////////////////////////////////////////////////////////////////////// @@ -1305,7 +1310,7 @@ Config::SetResourceConfigMode(const std::string& value) { Status Config::SetResourceConfigSearchResources(const std::string& value) { std::vector res_vec; - server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec); + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_RESOURCES_DELIMITER, res_vec); Status s = CheckResourceConfigSearchResources(res_vec); if (!s.ok()) { @@ -1317,13 +1322,16 @@ Config::SetResourceConfigSearchResources(const std::string& value) { } Status -Config::SetResourceConfigIndexBuildDevice(const std::string& value) { - Status s = CheckResourceConfigIndexBuildDevice(value); +Config::SetResourceConfigIndexBuildResources(const std::string &value) { + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_RESOURCES_DELIMITER, res_vec); + + Status s = CheckResourceConfigIndexBuildResources(res_vec); if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value); + SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_RESOURCES, value); return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 3ab0cd8053..0378a079fb 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -91,20 +91,18 @@ static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; static const char* CONFIG_RESOURCE = "resource_config"; static const char* CONFIG_RESOURCE_MODE = "mode"; static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; +static const char* CONFIG_RESOURCE_RESOURCES_DELIMITER = ","; static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources"; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ","; - #ifdef MILVUS_CPU_VERSION static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu"; #else static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0"; #endif - -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_RESOURCES = "index_build_resources"; #ifdef MILVUS_CPU_VERSION -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_RESOURCES_DEFAULT = "cpu"; #else -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_RESOURCES_DEFAULT = "gpu0"; #endif const int32_t CPU_DEVICE_ID = -1; @@ -190,7 +188,7 @@ class Config { Status CheckResourceConfigSearchResources(const std::vector& value); Status - CheckResourceConfigIndexBuildDevice(const std::string& value); + CheckResourceConfigIndexBuildResources(const std::vector& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -259,7 +257,7 @@ class Config { Status GetResourceConfigSearchResources(std::vector& value); Status - GetResourceConfigIndexBuildDevice(int32_t& value); + GetResourceConfigIndexBuildResources(std::vector& value); public: /* server config */ @@ -320,7 +318,7 @@ class Config { Status SetResourceConfigSearchResources(const std::string& value); Status - SetResourceConfigIndexBuildDevice(const std::string& value); + SetResourceConfigIndexBuildResources(const std::string& value); private: std::unordered_map> config_map_; diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index ec696ff3e0..080de77e17 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -182,7 +182,7 @@ ValidationUtil::ValidatePartitionTags(const std::vector& partition_ } Status -ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { +ValidationUtil::ValidateGpuIndex(int32_t gpu_index) { #ifdef MILVUS_GPU_VERSION int num_devices = 0; auto cuda_err = cudaGetDeviceCount(&num_devices); @@ -203,7 +203,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { } Status -ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { +ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) { #ifdef MILVUS_GPU_VERSION cudaDeviceProp deviceProp; diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 01801e295a..201ccef3bd 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -59,10 +59,10 @@ class ValidationUtil { ValidatePartitionTags(const std::vector& partition_tags); static Status - ValidateGpuIndex(uint32_t gpu_index); + ValidateGpuIndex(int32_t gpu_index); static Status - GetGpuMemory(uint32_t gpu_index, size_t& memory); + GetGpuMemory(int32_t gpu_index, size_t& memory); static Status ValidateIpAddress(const std::string& ip_address); diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 637273732d..37be36b7eb 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -272,29 +272,34 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { #else std::vector search_resources = {"cpu", "gpu0"}; #endif - std::vector res_vec; - std::string res_str; + std::vector search_res_vec; + std::string search_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( - search_resources, milvus::server::CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_str); - s = config.SetResourceConfigSearchResources(res_str); + search_resources, milvus::server::CONFIG_RESOURCE_RESOURCES_DELIMITER, search_res_str); + s = config.SetResourceConfigSearchResources(search_res_str); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigSearchResources(res_vec); + s = config.GetResourceConfigSearchResources(search_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < search_resources.size(); i++) { - ASSERT_TRUE(search_resources[i] == res_vec[i]); + ASSERT_TRUE(search_resources[i] == search_res_vec[i]); } #ifdef MILVUS_CPU_VERSION - int32_t resource_index_build_device = milvus::server::CPU_DEVICE_ID; - s = config.SetResourceConfigIndexBuildDevice("cpu"); + std::vector index_build_resources = {"cpu"}; #else - int32_t resource_index_build_device = 0; - s = config.SetResourceConfigIndexBuildDevice("gpu" + std::to_string(resource_index_build_device)); + std::vector index_build_resources = {"gpu0", "gpu1"}; #endif + std::vector index_build_res_vec; + std::string index_build_res_str; + milvus::server::StringHelpFunctions::MergeStringWithDelimeter( + index_build_resources, milvus::server::CONFIG_RESOURCE_RESOURCES_DELIMITER, index_build_res_str); + s = config.SetResourceConfigIndexBuildResources(index_build_res_str); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigIndexBuildDevice(int32_val); + s = config.GetResourceConfigIndexBuildResources(index_build_res_vec); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == resource_index_build_device); + for (size_t i = 0; i < index_build_resources.size(); i++) { + ASSERT_TRUE(index_build_resources[i] == index_build_res_vec[i]); + } } TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { @@ -418,9 +423,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetResourceConfigSearchResources("cpu"); ASSERT_TRUE(s.ok()); - s = config.SetResourceConfigIndexBuildDevice("gup2"); + s = config.SetResourceConfigIndexBuildResources("gup2"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigIndexBuildDevice("gpu16"); + s = config.SetResourceConfigIndexBuildResources("gpu16"); ASSERT_FALSE(s.ok()); } From 3eefd550591b092db769464f801824ab231c3751 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 18 Nov 2019 18:54:25 +0800 Subject: [PATCH 17/82] #397 sdk_simple return incorrect result --- CHANGELOG.md | 1 + core/src/sdk/examples/utils/Utils.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9240b357a0..eb4fba56c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#340 - Test cases run failed on 0.6.0 - \#353 - Rename config.h.in to version.h.in - \#374 - sdk_simple return empty result +- \#397 - sdk_simple return incorrect result ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/sdk/examples/utils/Utils.cpp b/core/src/sdk/examples/utils/Utils.cpp index d9dd950710..da5e854e9b 100644 --- a/core/src/sdk/examples/utils/Utils.cpp +++ b/core/src/sdk/examples/utils/Utils.cpp @@ -167,7 +167,7 @@ Utils::PrintSearchResult(const std::vector index++; std::cout << "No." << index << " vector " << search_id << " top " << topk << " search result:" << std::endl; for (size_t j = 0; j < topk; j++) { - size_t idx = i * nq + j; + size_t idx = i * topk + j; std::cout << "\t" << topk_query_result.ids[idx] << "\t" << topk_query_result.distances[idx] << std::endl; } } From 55ada7533c3918aa4e2480428aa1f71744043a5d Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Mon, 18 Nov 2019 19:10:37 +0800 Subject: [PATCH 18/82] #255 - Add ivfsq8 test report detailed version --- ...vus_ivfsq8_test_report_detailed_version.md | 210 +++++++++++++++++ ..._ivfsq8_test_report_detailed_version_cn.md | 211 ++++++++++++++++++ 2 files changed, 421 insertions(+) create mode 100644 docs/test_report/milvus_ivfsq8_test_report_detailed_version.md create mode 100644 docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md new file mode 100644 index 0000000000..4d6840430e --- /dev/null +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md @@ -0,0 +1,210 @@ +# milvus_ivfsq8_test_report_detailed_version + +## Summary + +This document contains the test reports of IVF_SQ8 index on Milvus single server. + + + +## Test objectives + +The time cost and recall when searching with different parameters. + + + +## Test method + +### Hardware/Software requirements + +Operating System: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +Memory: 503GB + +Docker version: 18.09 + +Nvidia Driver version: 430.34 + +Milvus version: 0.5.3 + +SDK interface: Python 3.6.8 + +Pymilvus version: 0.2.5 + + + +### Data model + +The data used in the tests are: + +- Data source: sift1b +- Data type: hdf5 + +For details on this dataset, you can check : http://corpus-texmex.irisa.fr/ . + + + +### Measures + +- Query Elapsed Time: Time cost (in seconds) to run a query. Variables that affect Query Elapsed Time: + + - nq (Number of queried vectors) + + > Note: In the query test of query elapsed time, we will test the following parameters with different values: + > + > nq - grouped by: [1, 5, 10, 200, 400, 600, 800, 1000], + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved . Variables that affect Recall: + + - nq (Number of queried vectors) + - topk (Top k result of a query) + + > Note: In the query test of recall, we will test the following parameters with different values: + > + > nq - grouped by: [1, 5, 10, 200, 400, 600, 800, 1000], + > + > topk - grouped by: [1, 10, 100] + + + +## Test reports + +### Test environment + +Data base: sift1b-1,000,000,000 vectors, 128-dimension + +Table Attributes + +- nlist: 16384 +- metric_type: L2 + +Query configuration + +- nprobe: 32 + +Milvus configuration + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 + +You can check the definition of Milvus configuration on https://milvus.io/docs/en/reference/milvus_config/. + +Test method + +Test the query elapsed time and recall with several parameters, and once only change one parameter. + +- Whether to restart Milvus after each query: No + + + +### Performance test + +#### Data query + +**Test result** + +Query Elapsed Time + +topk : 100 + +search_resources: gpu0, gpu1 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 15.57 | +| nq=10 | 15.80 | +| nq=200 | 15.72 | +| nq=400 | 15.94 | +| nq=600 | 16.58 | +| nq=800 | 16.71 | +| nq=1000 | 16.91 | + +When nq is 1000, the query time cost of a 128-dimension vector is around 17ms in GPU Mode. + + + +topk : 100 + +search_resources: cpu, gpu0 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 1.12 | +| nq=10 | 2.89 | +| nq=200 | 8.10 | +| nq=400 | 12.36 | +| nq=600 | 17.81 | +| nq=800 | 23.24 | +| nq=1000 | 27.41 | + +When nq is 1000, the query time cost of a 128-dimension vector is around 27ms in CPU Mode. + + + +**Conclusion** + +The query elapsed time in CPU Mode increases quickly with nq, while in GPU Mode query elapsed time increases much slower. When nq is small, CPU Mode consumes less time than GPU Mode. However, as nq becomes larger, GPU Mode shows its advantage against CPU Mode. + +The query elapsed time in GPU Mode consists of two parts: (1) index CPU-to-GPU copy time; (2) nprobe buckets search time. When nq is smaller than 500, index CPU-to-GPU copy time cannot be amortized efficiently, CPU Mode is a better choice; when nq is larger than 500, choosing GPU Mode is better. + +Compared with CPU, GPU has much more cores and stronger computing capability. When nq is large, it can better reflect GPU's advantages on computing. + + + +### Recall test + +**Test result** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +We use the ground_truth in sift1b dataset to calculate the recall of query results. + + + +Recall of GPU Mode + +search_resources: gpu0, gpu1 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=1 | 1.000 | 0.800 | 0.790 | +| nq=5 | 0.800 | 0.820 | 0.908 | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +Recall of CPU Mode + +search_resources: cpu, gpu0 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=1 | 1.000 | 0.800 | 0.790 | +| nq=5 | 0.800 | 0.820 | 0.908 | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +**Conclusion** + +As nq increases, the recall gradually stabilizes to over 93%. \ No newline at end of file diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md new file mode 100644 index 0000000000..4b5e18f291 --- /dev/null +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md @@ -0,0 +1,211 @@ +# milvus_ivfsq8_test_report_detailed_version_cn + +## 概述 + +本文描述了ivfsq8索引在milvus单机部署方式下的测试报告。 + + + +## 测试目标 + +参数不同情况下的查询时间和召回率。 + + + +## 测试方法 + +### 软硬件环境 + +操作系统: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +内存: 503GB + +Docker版本: 18.09 + +Nvidia Driver版本: 430.34 + +Milvus版本: 0.5.3 + +SDK接口: Python 3.6.8 + +Pymilvus版本: 0.2.5 + + + +### 数据模型 + +本测试中用到的主要数据: + +- 数据来源: sift1b +- 数据类型: hdf5 + +关于该数据集的详细信息请参考 : http://corpus-texmex.irisa.fr/ 。 + + + +### 测试指标 + +- Query Elapsed Time: 数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量: + + - nq (被查询向量的数量) + + > 备注：在向量查询测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [1, 5, 10, 200, 400, 600, 800, 1000]的数量分组。 + +- Recall: 实际返回的正确结果占总数之比 . 影响Recall的变量: + + - nq (被查询向量的数量) + - topk (单条查询中最相似的K个结果) + + > 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [1, 5, 10, 200, 400, 600, 800, 1000]的数量分组， + > + > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。 + + + +## 测试报告 + +### 测试环境 + +数据集: sift1b-1,000,000,000向量, 128维 + +表格属性： + +- nlist: 16384 +- metric_type: L2 + +查询设置： + +- nprobe: 32 + +Milvus设置： + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 + +你可以在 https://milvus.io/docs/en/reference/milvus_config/上查询Milvus设置的详细定义。 + +测试方法 + +通过一次仅改变一个参数的值，测试查询向量时间和召回率。 + +- 查询后是否重启Milvus：否 + + + +### 性能测试 + +#### 数据查询 + +测试结果 + +Query Elapsed Time + +topk : 100 + +search_resources: gpu0, gpu1 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 15.57 | +| nq=10 | 15.80 | +| nq=200 | 15.72 | +| nq=400 | 15.94 | +| nq=600 | 16.58 | +| nq=800 | 16.71 | +| nq=1000 | 16.91 | + +当nq为1000时，在GPU模式下查询一条128维向量需要耗时约17毫秒。 + + + +topk : 100 + +search_resources: cpu, gpu0 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 1.12 | +| nq=10 | 2.89 | +| nq=200 | 8.10 | +| nq=400 | 12.36 | +| nq=600 | 17.81 | +| nq=800 | 23.24 | +| nq=1000 | 27.41 | + +当nq为1000时，在GPU模式下查询一条128维向量需要耗时约27毫秒。 + + + +**总结** + +在CPU模式下查询耗时随nq的增长快速增大，而在GPU模式下查询耗时的增大则缓慢许多。当nq较小时，CPU模式比GPU模式耗时更少。但当nq足够大时，GPU模式则更具有优势。 + +在GPU模式下的查询耗时由两部分组成：（1）索引从CPU到GPU的拷贝时间；（2）所有分桶的查询时间。当nq小于500时，索引从CPU到GPU 的拷贝时间无法被有效均摊，此时CPU模式时一个更优的选择；当nq大于500时，选择GPU模式更合理。 + +和CPU相比，GPU具有更多的核数和更强的算力。当nq较大时，GPU在计算上的优势能被更好地被体现。 + + + +### 召回率测试 + +**测试结果** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +我们利用sift1b数据集中的ground_truth来计算查询结果的召回率。 + + + +Recall of GPU Mode + +search_resources: gpu0, gpu1 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=1 | 1.000 | 0.800 | 0.790 | +| nq=5 | 0.800 | 0.820 | 0.908 | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +Recall of CPU Mode + +search_resources: cpu, gpu0 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=1 | 1.000 | 0.800 | 0.790 | +| nq=5 | 0.800 | 0.820 | 0.908 | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +**总结** + +​ 随着nq的增大，召回率逐渐稳定至93%以上。 + From e70c8e029ef4534f15e5c0e18e302c4aa17e76f1 Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Mon, 18 Nov 2019 19:12:36 +0800 Subject: [PATCH 19/82] #255 - Add ivfsq8 test report detailed version, update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9240b357a0..4bcbb0b10a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md +- \#255 - Add ivfsq8 test report detailed version ## Task From 562b84a178afb90344e26bda3c021bff80e5f60c Mon Sep 17 00:00:00 2001 From: Shouyu Luo Date: Mon, 18 Nov 2019 23:03:59 +0800 Subject: [PATCH 20/82] commented out logs --- core/src/db/DBImpl.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index fc31846bd3..52cf1e668a 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -71,12 +71,12 @@ DBImpl::Start() { return Status::OK(); } - ENGINE_LOG_TRACE << "DB service start"; + // ENGINE_LOG_TRACE << "DB service start"; shutting_down_.store(false, std::memory_order_release); // for distribute version, some nodes are read only if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { - ENGINE_LOG_TRACE << "StartTimerTasks"; + // ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); } @@ -101,7 +101,7 @@ DBImpl::Stop() { meta_ptr_->CleanUp(); } - ENGINE_LOG_TRACE << "DB service stop"; + // ENGINE_LOG_TRACE << "DB service stop"; return Status::OK(); } @@ -506,7 +506,7 @@ DBImpl::StartMetricTask() { return; } - ENGINE_LOG_TRACE << "Start metric task"; + // ENGINE_LOG_TRACE << "Start metric task"; server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); @@ -532,7 +532,7 @@ DBImpl::StartMetricTask() { server::Metrics::GetInstance().GPUTemperature(); server::Metrics::GetInstance().CPUTemperature(); - ENGINE_LOG_TRACE << "Metric task finished"; + // ENGINE_LOG_TRACE << "Metric task finished"; } Status @@ -692,7 +692,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) { void DBImpl::BackgroundCompaction(std::set table_ids) { - ENGINE_LOG_TRACE << " Background compaction thread start"; + // ENGINE_LOG_TRACE << " Background compaction thread start"; Status status; for (auto& table_id : table_ids) { @@ -715,7 +715,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { } meta_ptr_->CleanUpFilesWithTTL(ttl); - ENGINE_LOG_TRACE << " Background compaction thread exit"; + // ENGINE_LOG_TRACE << " Background compaction thread exit"; } void @@ -748,7 +748,7 @@ DBImpl::StartBuildIndexTask(bool force) { void DBImpl::BackgroundBuildIndex() { - ENGINE_LOG_TRACE << "Background build index thread start"; + // ENGINE_LOG_TRACE << "Background build index thread start"; std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; @@ -771,7 +771,7 @@ DBImpl::BackgroundBuildIndex() { } } - ENGINE_LOG_TRACE << "Background build index thread exit"; + // ENGINE_LOG_TRACE << "Background build index thread exit"; } } // namespace engine From 8f64c44271785753f2fb34b34edb08598a950ba8 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Tue, 19 Nov 2019 09:16:58 +0800 Subject: [PATCH 21/82] [skip ci] Add note about using unbuntu docker --- install.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install.md b/install.md index 6711b41f76..98b9af4649 100644 --- a/install.md +++ b/install.md @@ -3,6 +3,9 @@ ## Software requirements - Ubuntu 18.04 or higher + + If your Ubuntu version is lower than 18.04, we recommend you to pull a [docker image of Ubuntu 18.04](https://docs.docker.com/install/linux/docker-ce/ubuntu/) as your compilation environment. + - CMake 3.12 or higher ##### For GPU version, you will also need: From 9fb536b502e6d17f1606c117b117bf55e1442a8b Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Tue, 19 Nov 2019 11:49:05 +0800 Subject: [PATCH 22/82] [skip ci] minor update --- install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.md b/install.md index 98b9af4649..f53586af36 100644 --- a/install.md +++ b/install.md @@ -4,7 +4,7 @@ - Ubuntu 18.04 or higher - If your Ubuntu version is lower than 18.04, we recommend you to pull a [docker image of Ubuntu 18.04](https://docs.docker.com/install/linux/docker-ce/ubuntu/) as your compilation environment. + If your operating system is not Ubuntu 18.04 or higher, we recommend you to pull a [docker image of Ubuntu 18.04](https://docs.docker.com/install/linux/docker-ce/ubuntu/) as your compilation environment. - CMake 3.12 or higher From 7ad5c514970497a4583098f07756d7c25176eea8 Mon Sep 17 00:00:00 2001 From: wxyu Date: Tue, 19 Nov 2019 12:29:05 +0800 Subject: [PATCH 23/82] Add virtual method Init() in Pass abstract class close #404 --- CHANGELOG.md | 1 + core/src/scheduler/SchedInst.cpp | 1 + core/src/scheduler/optimizer/HybridPass.cpp | 4 ++++ core/src/scheduler/optimizer/HybridPass.h | 3 +++ core/src/scheduler/optimizer/LargeSQ8HPass.cpp | 3 ++- core/src/scheduler/optimizer/LargeSQ8HPass.h | 5 ++++- core/src/scheduler/optimizer/OnlyCPUPass.cpp | 4 ++++ core/src/scheduler/optimizer/OnlyCPUPass.h | 3 +++ core/src/scheduler/optimizer/OnlyGPUPass.cpp | 4 ++++ core/src/scheduler/optimizer/OnlyGPUPass.h | 3 +++ core/src/scheduler/optimizer/Optimizer.cpp | 12 ++++++------ core/src/scheduler/optimizer/Optimizer.h | 4 ++-- core/src/scheduler/optimizer/Pass.h | 5 ++--- 13 files changed, 39 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bcbb0b10a..a87e4bec63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md - \#255 - Add ivfsq8 test report detailed version +- \#404 - Add virtual method Init() in Pass abstract class ## Task diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 61e0c09759..9bd4df944f 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -84,6 +84,7 @@ load_simple_config() { void StartSchedulerService() { load_simple_config(); + OptimizerInst::GetInstance()->Init(); ResMgrInst::GetInstance()->Start(); SchedInst::GetInstance()->Start(); JobMgrInst::GetInstance()->Start(); diff --git a/core/src/scheduler/optimizer/HybridPass.cpp b/core/src/scheduler/optimizer/HybridPass.cpp index d63fc2e819..8bf0d8990d 100644 --- a/core/src/scheduler/optimizer/HybridPass.cpp +++ b/core/src/scheduler/optimizer/HybridPass.cpp @@ -23,6 +23,10 @@ namespace milvus { namespace scheduler { +void +HybridPass::Init() { +} + bool HybridPass::Run(const TaskPtr& task) { // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu diff --git a/core/src/scheduler/optimizer/HybridPass.h b/core/src/scheduler/optimizer/HybridPass.h index 0d02a8bda9..f84a0884f2 100644 --- a/core/src/scheduler/optimizer/HybridPass.h +++ b/core/src/scheduler/optimizer/HybridPass.h @@ -37,6 +37,9 @@ class HybridPass : public Pass { HybridPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; }; diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp index b9784e3c0a..89bd85ec1f 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp @@ -27,7 +27,8 @@ namespace milvus { namespace scheduler { -LargeSQ8HPass::LargeSQ8HPass() { +void +LargeSQ8HPass::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/LargeSQ8HPass.h index 9d135d413a..1afd240c0f 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.h +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.h @@ -35,9 +35,12 @@ namespace scheduler { class LargeSQ8HPass : public Pass { public: - LargeSQ8HPass(); + LargeSQ8HPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.cpp b/core/src/scheduler/optimizer/OnlyCPUPass.cpp index 238a91a82c..392ae7940b 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.cpp +++ b/core/src/scheduler/optimizer/OnlyCPUPass.cpp @@ -24,6 +24,10 @@ namespace milvus { namespace scheduler { +void +OnlyCPUPass::Init() { +} + bool OnlyCPUPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::SearchTask) diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.h b/core/src/scheduler/optimizer/OnlyCPUPass.h index 76b42e3766..253775f77e 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.h +++ b/core/src/scheduler/optimizer/OnlyCPUPass.h @@ -37,6 +37,9 @@ class OnlyCPUPass : public Pass { OnlyCPUPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; }; diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.cpp b/core/src/scheduler/optimizer/OnlyGPUPass.cpp index 2a72f9757e..69a8366bb2 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.cpp +++ b/core/src/scheduler/optimizer/OnlyGPUPass.cpp @@ -27,6 +27,10 @@ namespace scheduler { OnlyGPUPass::OnlyGPUPass(bool has_cpu) : has_cpu_(has_cpu) { } +void +OnlyGPUPass::Init() { +} + bool OnlyGPUPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::SearchTask || has_cpu_) diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.h b/core/src/scheduler/optimizer/OnlyGPUPass.h index 10d909d30e..c7a83fea30 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.h +++ b/core/src/scheduler/optimizer/OnlyGPUPass.h @@ -37,6 +37,9 @@ class OnlyGPUPass : public Pass { explicit OnlyGPUPass(bool has_cpu); public: + void + Init() override; + bool Run(const TaskPtr& task) override; diff --git a/core/src/scheduler/optimizer/Optimizer.cpp b/core/src/scheduler/optimizer/Optimizer.cpp index 46f24ea712..c5fa311a27 100644 --- a/core/src/scheduler/optimizer/Optimizer.cpp +++ b/core/src/scheduler/optimizer/Optimizer.cpp @@ -20,12 +20,12 @@ namespace milvus { namespace scheduler { -// void -// Optimizer::Init() { -// for (auto& pass : pass_list_) { -// pass->Init(); -// } -// } +void +Optimizer::Init() { + for (auto& pass : pass_list_) { + pass->Init(); + } +} bool Optimizer::Run(const TaskPtr& task) { diff --git a/core/src/scheduler/optimizer/Optimizer.h b/core/src/scheduler/optimizer/Optimizer.h index bfabbf7de3..68b519e115 100644 --- a/core/src/scheduler/optimizer/Optimizer.h +++ b/core/src/scheduler/optimizer/Optimizer.h @@ -38,8 +38,8 @@ class Optimizer { explicit Optimizer(std::vector pass_list) : pass_list_(std::move(pass_list)) { } - // void - // Init(); + void + Init(); bool Run(const TaskPtr& task); diff --git a/core/src/scheduler/optimizer/Pass.h b/core/src/scheduler/optimizer/Pass.h index 016b05e457..36a36a1df5 100644 --- a/core/src/scheduler/optimizer/Pass.h +++ b/core/src/scheduler/optimizer/Pass.h @@ -34,9 +34,8 @@ namespace scheduler { class Pass { public: - // virtual void - // Init() { - // } + virtual void + Init() = 0; virtual bool Run(const TaskPtr& task) = 0; From 31aa89e1318c3e8b9b2e1cee220b20b7fc42370b Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Tue, 19 Nov 2019 12:59:54 +0800 Subject: [PATCH 24/82] Change ivfsq8 test report detailed version --- .../milvus_ivfsq8_test_report_detailed_version.md | 4 ---- .../milvus_ivfsq8_test_report_detailed_version_cn.md | 6 +----- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md index 4d6840430e..dc1bbb3fd0 100644 --- a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md @@ -177,8 +177,6 @@ search_resources: gpu0, gpu1 | nq/topk | topk=1 | topk=10 | topk=100 | | :-----: | :----: | :-----: | :------: | -| nq=1 | 1.000 | 0.800 | 0.790 | -| nq=5 | 0.800 | 0.820 | 0.908 | | nq=10 | 0.900 | 0.910 | 0.939 | | nq=200 | 0.955 | 0.941 | 0.929 | | nq=400 | 0.958 | 0.944 | 0.932 | @@ -194,8 +192,6 @@ search_resources: cpu, gpu0 | nq/topk | topk=1 | topk=10 | topk=100 | | :-----: | :----: | :-----: | :------: | -| nq=1 | 1.000 | 0.800 | 0.790 | -| nq=5 | 0.800 | 0.820 | 0.908 | | nq=10 | 0.900 | 0.910 | 0.939 | | nq=200 | 0.955 | 0.941 | 0.929 | | nq=400 | 0.958 | 0.944 | 0.932 | diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md index 4b5e18f291..bf43d2e0e8 100644 --- a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md @@ -177,8 +177,6 @@ search_resources: gpu0, gpu1 | nq/topk | topk=1 | topk=10 | topk=100 | | :-----: | :----: | :-----: | :------: | -| nq=1 | 1.000 | 0.800 | 0.790 | -| nq=5 | 0.800 | 0.820 | 0.908 | | nq=10 | 0.900 | 0.910 | 0.939 | | nq=200 | 0.955 | 0.941 | 0.929 | | nq=400 | 0.958 | 0.944 | 0.932 | @@ -194,8 +192,6 @@ search_resources: cpu, gpu0 | nq/topk | topk=1 | topk=10 | topk=100 | | :-----: | :----: | :-----: | :------: | -| nq=1 | 1.000 | 0.800 | 0.790 | -| nq=5 | 0.800 | 0.820 | 0.908 | | nq=10 | 0.900 | 0.910 | 0.939 | | nq=200 | 0.955 | 0.941 | 0.929 | | nq=400 | 0.958 | 0.944 | 0.932 | @@ -207,5 +203,5 @@ search_resources: cpu, gpu0 **总结** -​ 随着nq的增大，召回率逐渐稳定至93%以上。 +随着nq的增大，召回率逐渐稳定至93%以上。 From 2707e07ccd36e47d1ff60d3be9e95832a6942b28 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 19 Nov 2019 14:31:32 +0800 Subject: [PATCH 25/82] Fix Deployed GPU-version docker failed in CI --- ci/jenkins/step/deploySingle2Dev.groovy | 6 +----- ci/jenkins/step/singleDevNightlyTest.groovy | 6 +----- ci/jenkins/step/singleDevTest.groovy | 6 +----- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index f1daaf22ec..7b479ff44a 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,11 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - if ("${env.BINRARY_VERSION}" == "gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } else { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/filebeat/values.yaml --namespace milvus ." - } + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index d357badfd3..e27bc6c0a5 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,11 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - if ("${env.BINRARY_VERSION}" == "gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } else { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index c1de5907b0..08812ae01f 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,11 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // if ("${env.BINRARY_VERSION}" == "gpu") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - // } else { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - // } + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { From 0d1923c61f53ea6ec77c98356e6c8301317d9b74 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Tue, 19 Nov 2019 15:49:40 +0800 Subject: [PATCH 26/82] #346 update gpu resource config APIs --- core/conf/server_cpu_config.template | 15 +- core/conf/server_gpu_config.template | 15 +- core/src/cache/GpuCacheMgr.cpp | 4 +- core/src/server/Config.cpp | 462 ++++++++++++++------------- core/src/server/Config.h | 87 +++-- core/src/utils/ValidationUtil.cpp | 4 +- core/src/utils/ValidationUtil.h | 4 +- core/unittest/server/test_config.cpp | 116 ++++--- 8 files changed, 357 insertions(+), 350 deletions(-) diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 6c95126390..ae942f351e 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -27,9 +27,7 @@ metric_config: port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] cache_config: - - cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] + cpu_cache_capacity: 16 # GB, size of CPU memory used for cache, must be a positive integer cache_insert_data: false # whether to load inserted data into cache, must be a boolean engine_config: @@ -37,7 +35,10 @@ engine_config: # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only -resource_config: - search_resources: # define the device used for search computation - - cpu - index_build_device: cpu # CPU used for building index +gpu_resource_config: + enable_gpu: true # whether to enable GPU resources + cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer + search_resources: # define the GPU devices used for search computation, must be in format gpux + - gpu0 + build_index_resources: # define the GPU devices used for index building, must be in format gpux + - gpu0 \ No newline at end of file diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 154db5d134..92a294d483 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -27,10 +27,7 @@ metric_config: port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] cache_config: - cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] - gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer - gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] + cpu_cache_capacity: 16 # GB, size of CPU memory used for cache, must be a positive integer cache_insert_data: false # whether to load inserted data into cache, must be a boolean engine_config: @@ -38,8 +35,10 @@ engine_config: # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only -resource_config: - search_resources: # define the devices used for search computation, must be in format: cpu or gpux - - cpu +gpu_resource_config: + enable_gpu: false # whether to enable GPU resources + cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer + search_resources: # define the GPU devices used for search computation, must be in format gpux - gpu0 - index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu or gpux + build_index_resources: # define the GPU devices used for index building, must be in format gpux + - gpu0 \ No newline at end of file diff --git a/core/src/cache/GpuCacheMgr.cpp b/core/src/cache/GpuCacheMgr.cpp index d862bc0393..72229527fa 100644 --- a/core/src/cache/GpuCacheMgr.cpp +++ b/core/src/cache/GpuCacheMgr.cpp @@ -37,7 +37,7 @@ GpuCacheMgr::GpuCacheMgr() { Status s; int64_t gpu_cache_cap; - s = config.GetCacheConfigGpuCacheCapacity(gpu_cache_cap); + s = config.GetGpuResourceConfigCacheCapacity(gpu_cache_cap); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } @@ -45,7 +45,7 @@ GpuCacheMgr::GpuCacheMgr() { cache_ = std::make_shared>(cap, 1UL << 32); float gpu_mem_threshold; - s = config.GetCacheConfigGpuCacheThreshold(gpu_mem_threshold); + s = config.GetGpuResourceConfigCacheThreshold(gpu_mem_threshold); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index f130e73a85..017f1641bd 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -163,20 +163,6 @@ Config::ValidateConfig() { return s; } -#ifdef MILVUS_GPU_VERSION - int64_t cache_gpu_cache_capacity; - s = GetCacheConfigGpuCacheCapacity(cache_gpu_cache_capacity); - if (!s.ok()) { - return s; - } - - float cache_gpu_cache_threshold; - s = GetCacheConfigGpuCacheThreshold(cache_gpu_cache_threshold); - if (!s.ok()) { - return s; - } -#endif - bool cache_insert_data; s = GetCacheConfigCacheInsertData(cache_insert_data); if (!s.ok()) { @@ -202,25 +188,39 @@ Config::ValidateConfig() { return s; } - /* resource config */ - std::string resource_mode; - s = GetResourceConfigMode(resource_mode); + /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION + bool resource_enable_gpu; + s = GetGpuResourceConfigEnableGpu(resource_enable_gpu); if (!s.ok()) { return s; } - std::vector search_resources; - s = GetResourceConfigSearchResources(search_resources); + int64_t resource_cache_capacity; + s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity); if (!s.ok()) { return s; } - int32_t resource_index_build_device; - s = GetResourceConfigIndexBuildDevice(resource_index_build_device); + float resource_cache_threshold; + s = GetGpuResourceConfigCacheThreshold(resource_cache_threshold); if (!s.ok()) { return s; } + std::vector search_resources; + s = GetGpuResourceConfigSearchResources(search_resources); + if (!s.ok()) { + return s; + } + + std::vector index_build_resources; + s = GetGpuResourceConfigBuildIndexResources(index_build_resources); + if (!s.ok()) { + return s; + } +#endif + return Status::OK(); } @@ -307,18 +307,6 @@ Config::ResetDefaultConfig() { return s; } -#ifdef MILVUS_GPU_VERSION - s = SetCacheConfigGpuCacheCapacity(CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetCacheConfigGpuCacheThreshold(CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); - if (!s.ok()) { - return s; - } -#endif - s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); if (!s.ok()) { return s; @@ -340,22 +328,34 @@ Config::ResetDefaultConfig() { return s; } - /* resource config */ - s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT); + /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION + s = SetGpuResourceConfigEnableGpu(CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT); if (!s.ok()) { return s; } - s = SetResourceConfigSearchResources(CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); + s = SetGpuResourceConfigCacheCapacity(CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); if (!s.ok()) { return s; } - s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + s = SetGpuResourceConfigCacheThreshold(CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT); if (!s.ok()) { return s; } + s = SetGpuResourceConfigSearchResources(CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT); + if (!s.ok()) { + return s; + } + + s = SetGpuResourceConfigBuildIndexResources(CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT); + if (!s.ok()) { + return s; + } +#endif + return Status::OK(); } @@ -377,7 +377,7 @@ Config::PrintAll() { PrintConfigSection(CONFIG_CACHE); PrintConfigSection(CONFIG_METRIC); PrintConfigSection(CONFIG_ENGINE); - PrintConfigSection(CONFIG_RESOURCE); + PrintConfigSection(CONFIG_GPU_RESOURCE); } //////////////////////////////////////////////////////////////////////////////// @@ -591,52 +591,6 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { return Status::OK(); } -Status -Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - uint64_t gpu_cache_capacity = std::stoi(value) * GB; - int device_id; - Status s = GetResourceConfigIndexBuildDevice(device_id); - if (!s.ok()) { - return s; - } - - size_t gpu_memory; - if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { - std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); - return Status(SERVER_UNEXPECTED_ERROR, msg); - } else if (gpu_cache_capacity >= gpu_memory) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { - std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; - } - } - return Status::OK(); -} - -Status -Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) { - if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { - std::string msg = "Invalid gpu cache threshold: " + value + - ". Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - float gpu_cache_threshold = std::stof(value); - if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { - std::string msg = "Invalid gpu cache threshold: " + value + - ". Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } - return Status::OK(); -} - Status Config::CheckCacheConfigCacheInsertData(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { @@ -687,56 +641,99 @@ Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { } Status -Config::CheckResourceConfigMode(const std::string& value) { - if (value != "simple") { - std::string msg = "Invalid resource mode: " + value + ". Possible reason: resource_config.mode is invalid."; +Config::CheckGpuResourceConfigEnableGpu(const std::string& value) { + if (!ValidationUtil::ValidateStringIsBool(value).ok()) { + std::string msg = "Invalid gpu resource config: " + value + + ". Possible reason: gpu_resource_config.enable_gpu is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } Status -CheckResource(const std::string& value) { +Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: gpu_resource_config.cache_capacity is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + uint64_t gpu_cache_capacity = std::stoi(value) * GB; + std::vector gpu_ids; + Status s = GetGpuResourceConfigBuildIndexResources(gpu_ids); + if (!s.ok()) { + return s; + } + + for (int32_t gpu_id : gpu_ids) { + size_t gpu_memory; + if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) { + std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id); + return Status(SERVER_UNEXPECTED_ERROR, msg); + } else if (gpu_cache_capacity >= gpu_memory) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: gpu_resource_config.cache_capacity exceeds GPU memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { + std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + } + } + } + return Status::OK(); +} + +Status +Config::CheckGpuResourceConfigCacheThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { + std::string msg = "Invalid gpu cache threshold: " + value + + ". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + float gpu_cache_threshold = std::stof(value); + if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { + std::string msg = "Invalid gpu cache threshold: " + value + + ". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + } + return Status::OK(); +} + +Status +CheckGpuResource(const std::string& value) { std::string s = value; std::transform(s.begin(), s.end(), s.begin(), ::tolower); -#ifdef MILVUS_CPU_VERSION - if (s != "cpu") { - return Status(SERVER_INVALID_ARGUMENT, "Invalid CPU resource: " + s); - } -#else - const std::regex pat("cpu|gpu(\\d+)"); + const std::regex pat("gpu(\\d+)"); std::smatch m; if (!std::regex_match(s, m, pat)) { - std::string msg = "Invalid search resource: " + value + - ". Possible reason: resource_config.search_resources is not in the format of cpux or gpux"; + std::string msg = "Invalid gpu resource: " + value + + ". Possible reason: gpu_resource_config is not in the format of cpux or gpux"; return Status(SERVER_INVALID_ARGUMENT, msg); } if (s.compare(0, 3, "gpu") == 0) { int32_t gpu_index = std::stoi(s.substr(3)); if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - std::string msg = "Invalid search resource: " + value + - ". Possible reason: resource_config.search_resources does not match your hardware."; + std::string msg = "Invalid gpu resource: " + value + + ". Possible reason: gpu_resource_config does not match with the hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } } -#endif return Status::OK(); } Status -Config::CheckResourceConfigSearchResources(const std::vector& value) { +Config::CheckGpuResourceConfigSearchResources(const std::vector& value) { if (value.empty()) { std::string msg = - "Invalid search resource. " - "Possible reason: resource_config.search_resources is empty."; + "Invalid gpu search resource. " + "Possible reason: gpu_resource_config.search_resources is empty."; return Status(SERVER_INVALID_ARGUMENT, msg); } for (auto& resource : value) { - auto status = CheckResource(resource); + auto status = CheckGpuResource(resource); if (!status.ok()) { return Status(SERVER_INVALID_ARGUMENT, status.message()); } @@ -745,10 +742,19 @@ Config::CheckResourceConfigSearchResources(const std::vector& value } Status -Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { - auto status = CheckResource(value); - if (!status.ok()) { - return Status(SERVER_INVALID_ARGUMENT, status.message()); +Config::CheckGpuResourceConfigBuildIndexResources(const std::vector& value) { + if (value.empty()) { + std::string msg = + "Invalid gpu build index resource. " + "Possible reason: gpu_resource_config.build_index_resources is empty."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + + for (auto& resource : value) { + auto status = CheckGpuResource(resource); + if (!status.ok()) { + return Status(SERVER_INVALID_ARGUMENT, status.message()); + } } return Status::OK(); } @@ -855,7 +861,6 @@ Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } @@ -868,7 +873,6 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } @@ -880,7 +884,6 @@ Config::GetDBConfigInsertBufferSize(int32_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } @@ -898,7 +901,6 @@ Config::GetMetricConfigEnableMonitor(bool& value) { if (!s.ok()) { return s; } - std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); @@ -924,7 +926,6 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } @@ -937,33 +938,6 @@ Config::GetCacheConfigCpuCacheThreshold(float& value) { if (!s.ok()) { return s; } - - value = std::stof(str); - return Status::OK(); -} - -Status -Config::GetCacheConfigGpuCacheCapacity(int64_t& value) { - std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); - Status s = CheckCacheConfigGpuCacheCapacity(str); - if (!s.ok()) { - return s; - } - - value = std::stoi(str); - return Status::OK(); -} - -Status -Config::GetCacheConfigGpuCacheThreshold(float& value) { - std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); - Status s = CheckCacheConfigGpuCacheThreshold(str); - if (!s.ok()) { - return s; - } - value = std::stof(str); return Status::OK(); } @@ -976,7 +950,6 @@ Config::GetCacheConfigCacheInsertData(bool& value) { if (!s.ok()) { return s; } - std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); @@ -990,7 +963,6 @@ Config::GetEngineConfigUseBlasThreshold(int32_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } @@ -1002,7 +974,6 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } @@ -1015,41 +986,113 @@ Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); return Status::OK(); } Status -Config::GetResourceConfigMode(std::string& value) { - value = GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - return CheckResourceConfigMode(value); -} - -Status -Config::GetResourceConfigSearchResources(std::vector& value) { +Config::GetGpuResourceConfigEnableGpu(bool& value) { std::string str = - GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, - CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); - server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value); - return CheckResourceConfigSearchResources(value); -} - -Status -Config::GetResourceConfigIndexBuildDevice(int32_t& value) { - std::string str = - GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); - Status s = CheckResourceConfigIndexBuildDevice(str); + GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE_GPU, CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT); + Status s = CheckGpuResourceConfigEnableGpu(str); if (!s.ok()) { return s; } + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + value = (str == "true" || str == "on" || str == "yes" || str == "1"); + return Status::OK(); +} - if (str == "cpu") { - value = CPU_DEVICE_ID; - } else { - value = std::stoi(str.substr(3)); +Status +Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { + bool enable_gpu = false; + Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + if (!s.ok()) { + return s; } + if (!enable_gpu) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, + CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); + s = CheckGpuResourceConfigCacheCapacity(str); + if (!s.ok()) { + return s; + } + value = std::stoi(str); + return Status::OK(); +} +Status +Config::GetGpuResourceConfigCacheThreshold(float& value) { + bool enable_gpu = false; + Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + if (!s.ok()) { + return s; + } + if (!enable_gpu) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, + CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT); + s = CheckGpuResourceConfigCacheThreshold(str); + if (!s.ok()) { + return s; + } + value = std::stof(str); + return Status::OK(); +} + +Status +Config::GetGpuResourceConfigSearchResources(std::vector& value) { + bool enable_gpu = false; + Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + if (!s.ok()) { + return s; + } + if (!enable_gpu) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, + CONFIG_GPU_RESOURCE_DELIMITER, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT); + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + s = CheckGpuResourceConfigSearchResources(res_vec); + if (!s.ok()) { + return s; + } + for (std::string& res : res_vec) { + value.push_back(std::stoi(res.substr(3))); + } + return Status::OK(); +} + +Status +Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { + bool enable_gpu = false; + Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + if (!s.ok()) { + return s; + } + if (!enable_gpu) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = + GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, + CONFIG_GPU_RESOURCE_DELIMITER, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT); + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + s = CheckGpuResourceConfigBuildIndexResources(res_vec); + if (!s.ok()) { + return s; + } + for (std::string& res : res_vec) { + value.push_back(std::stoi(res.substr(3))); + } return Status::OK(); } @@ -1061,7 +1104,6 @@ Config::SetServerConfigAddress(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); return Status::OK(); } @@ -1072,7 +1114,6 @@ Config::SetServerConfigPort(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); return Status::OK(); } @@ -1083,7 +1124,6 @@ Config::SetServerConfigDeployMode(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); return Status::OK(); } @@ -1094,7 +1134,6 @@ Config::SetServerConfigTimeZone(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); return Status::OK(); } @@ -1106,7 +1145,6 @@ Config::SetDBConfigPrimaryPath(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); return Status::OK(); } @@ -1117,7 +1155,6 @@ Config::SetDBConfigSecondaryPath(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); return Status::OK(); } @@ -1128,7 +1165,6 @@ Config::SetDBConfigBackendUrl(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); return Status::OK(); } @@ -1139,7 +1175,6 @@ Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); return Status::OK(); } @@ -1150,7 +1185,6 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); return Status::OK(); } @@ -1161,7 +1195,6 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); return Status::OK(); } @@ -1173,7 +1206,6 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); return Status::OK(); } @@ -1184,7 +1216,6 @@ Config::SetMetricConfigCollector(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); return Status::OK(); } @@ -1195,7 +1226,6 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); return Status::OK(); } @@ -1207,7 +1237,6 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); return Status::OK(); } @@ -1218,40 +1247,16 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); return Status::OK(); } -Status -Config::SetCacheConfigGpuCacheCapacity(const std::string& value) { - Status s = CheckCacheConfigGpuCacheCapacity(value); - if (!s.ok()) { - return s; - } - - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); - return Status::OK(); -} - -Status -Config::SetCacheConfigGpuCacheThreshold(const std::string& value) { - Status s = CheckCacheConfigGpuCacheThreshold(value); - if (!s.ok()) { - return s; - } - - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); - return Status::OK(); -} - Status Config::SetCacheConfigCacheInsertData(const std::string& value) { Status s = CheckCacheConfigCacheInsertData(value); if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); return Status::OK(); } @@ -1263,7 +1268,6 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); return Status::OK(); } @@ -1274,7 +1278,6 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); return Status::OK(); } @@ -1285,45 +1288,62 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value); return Status::OK(); } -/* resource config */ +/* gpu resource config */ Status -Config::SetResourceConfigMode(const std::string& value) { - Status s = CheckResourceConfigMode(value); +Config::SetGpuResourceConfigEnableGpu(const std::string& value) { + Status s = CheckGpuResourceConfigEnableGpu(value); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE_GPU, value); return Status::OK(); } Status -Config::SetResourceConfigSearchResources(const std::string& value) { +Config::SetGpuResourceConfigCacheCapacity(const std::string& value) { + Status s = CheckGpuResourceConfigCacheCapacity(value); + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, value); + return Status::OK(); +} + +Status +Config::SetGpuResourceConfigCacheThreshold(const std::string& value) { + Status s = CheckGpuResourceConfigCacheThreshold(value); + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetGpuResourceConfigSearchResources(const std::string& value) { std::vector res_vec; - server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec); - - Status s = CheckResourceConfigSearchResources(res_vec); + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + Status s = CheckGpuResourceConfigSearchResources(res_vec); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, value); return Status::OK(); } Status -Config::SetResourceConfigIndexBuildDevice(const std::string& value) { - Status s = CheckResourceConfigIndexBuildDevice(value); +Config::SetGpuResourceConfigBuildIndexResources(const std::string& value) { + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + Status s = CheckGpuResourceConfigBuildIndexResources(res_vec); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, value); return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 3ab0cd8053..0442ae0626 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -59,12 +59,8 @@ static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; static const char* CONFIG_CACHE = "cache_config"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "16"; -static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; -static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT = "4"; -static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_mem_threshold"; +static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_cache_threshold"; static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.85"; -static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD = "gpu_mem_threshold"; -static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; @@ -87,26 +83,23 @@ static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD = "gpu_search_threshold"; static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; -/* resource config */ -static const char* CONFIG_RESOURCE = "resource_config"; -static const char* CONFIG_RESOURCE_MODE = "mode"; -static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources"; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ","; - -#ifdef MILVUS_CPU_VERSION -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu"; +/* gpu resource config */ +static const char* CONFIG_GPU_RESOURCE = "gpu_resource_config"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU = "enable_gpu"; +#ifdef MILVUS_GPU_VERSION +static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "true"; #else -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "false"; #endif - -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device"; -#ifdef MILVUS_CPU_VERSION -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu"; -#else -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0"; -#endif -const int32_t CPU_DEVICE_ID = -1; +static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY = "cache_capacity"; +static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "4"; +static const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD = "cache_threshold"; +static const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_GPU_RESOURCE_DELIMITER = ","; +static const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES = "search_resources"; +static const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT = "gpu0"; +static const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES = "build_index_resources"; +static const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT = "gpu0"; class Config { public: @@ -170,10 +163,6 @@ class Config { Status CheckCacheConfigCpuCacheThreshold(const std::string& value); Status - CheckCacheConfigGpuCacheCapacity(const std::string& value); - Status - CheckCacheConfigGpuCacheThreshold(const std::string& value); - Status CheckCacheConfigCacheInsertData(const std::string& value); /* engine config */ @@ -184,13 +173,17 @@ class Config { Status CheckEngineConfigGpuSearchThreshold(const std::string& value); - /* resource config */ + /* gpu resource config */ Status - CheckResourceConfigMode(const std::string& value); + CheckGpuResourceConfigEnableGpu(const std::string& value); Status - CheckResourceConfigSearchResources(const std::vector& value); + CheckGpuResourceConfigCacheCapacity(const std::string& value); Status - CheckResourceConfigIndexBuildDevice(const std::string& value); + CheckGpuResourceConfigCacheThreshold(const std::string& value); + Status + CheckGpuResourceConfigSearchResources(const std::vector& value); + Status + CheckGpuResourceConfigBuildIndexResources(const std::vector& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -239,10 +232,6 @@ class Config { Status GetCacheConfigCpuCacheThreshold(float& value); Status - GetCacheConfigGpuCacheCapacity(int64_t& value); - Status - GetCacheConfigGpuCacheThreshold(float& value); - Status GetCacheConfigCacheInsertData(bool& value); /* engine config */ @@ -253,13 +242,17 @@ class Config { Status GetEngineConfigGpuSearchThreshold(int32_t& value); - /* resource config */ + /* gpu resource config */ Status - GetResourceConfigMode(std::string& value); + GetGpuResourceConfigEnableGpu(bool& value); Status - GetResourceConfigSearchResources(std::vector& value); + GetGpuResourceConfigCacheCapacity(int64_t& value); Status - GetResourceConfigIndexBuildDevice(int32_t& value); + GetGpuResourceConfigCacheThreshold(float& value); + Status + GetGpuResourceConfigSearchResources(std::vector& value); + Status + GetGpuResourceConfigBuildIndexResources(std::vector& value); public: /* server config */ @@ -300,10 +293,6 @@ class Config { Status SetCacheConfigCpuCacheThreshold(const std::string& value); Status - SetCacheConfigGpuCacheCapacity(const std::string& value); - Status - SetCacheConfigGpuCacheThreshold(const std::string& value); - Status SetCacheConfigCacheInsertData(const std::string& value); /* engine config */ @@ -314,13 +303,17 @@ class Config { Status SetEngineConfigGpuSearchThreshold(const std::string& value); - /* resource config */ + /* gpu resource config */ Status - SetResourceConfigMode(const std::string& value); + SetGpuResourceConfigEnableGpu(const std::string& value); Status - SetResourceConfigSearchResources(const std::string& value); + SetGpuResourceConfigCacheCapacity(const std::string& value); Status - SetResourceConfigIndexBuildDevice(const std::string& value); + SetGpuResourceConfigCacheThreshold(const std::string& value); + Status + SetGpuResourceConfigSearchResources(const std::string& value); + Status + SetGpuResourceConfigBuildIndexResources(const std::string& value); private: std::unordered_map> config_map_; diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index ec696ff3e0..080de77e17 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -182,7 +182,7 @@ ValidationUtil::ValidatePartitionTags(const std::vector& partition_ } Status -ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { +ValidationUtil::ValidateGpuIndex(int32_t gpu_index) { #ifdef MILVUS_GPU_VERSION int num_devices = 0; auto cuda_err = cudaGetDeviceCount(&num_devices); @@ -203,7 +203,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { } Status -ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { +ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) { #ifdef MILVUS_GPU_VERSION cudaDeviceProp deviceProp; diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 01801e295a..201ccef3bd 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -59,10 +59,10 @@ class ValidationUtil { ValidatePartitionTags(const std::vector& partition_tags); static Status - ValidateGpuIndex(uint32_t gpu_index); + ValidateGpuIndex(int32_t gpu_index); static Status - GetGpuMemory(uint32_t gpu_index, size_t& memory); + GetGpuMemory(int32_t gpu_index, size_t& memory); static Status ValidateIpAddress(const std::string& ip_address); diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 637273732d..387d321b85 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -216,21 +216,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.GetCacheConfigCpuCacheThreshold(float_val); ASSERT_TRUE(float_val == cache_cpu_cache_threshold); -#ifdef MILVUS_GPU_VERSION - int64_t cache_gpu_cache_capacity = 1; - s = config.SetCacheConfigGpuCacheCapacity(std::to_string(cache_gpu_cache_capacity)); - ASSERT_TRUE(s.ok()); - s = config.GetCacheConfigGpuCacheCapacity(int64_val); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int64_val == cache_gpu_cache_capacity); - - float cache_gpu_cache_threshold = 0.2; - s = config.SetCacheConfigGpuCacheThreshold(std::to_string(cache_gpu_cache_threshold)); - ASSERT_TRUE(s.ok()); - s = config.GetCacheConfigGpuCacheThreshold(float_val); - ASSERT_TRUE(float_val == cache_gpu_cache_threshold); -#endif - bool cache_insert_data = true; s = config.SetCacheConfigCacheInsertData(std::to_string(cache_insert_data)); ASSERT_TRUE(s.ok()); @@ -259,42 +244,54 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(s.ok()); ASSERT_TRUE(int32_val == engine_gpu_search_threshold); - /* resource config */ - std::string resource_mode = "simple"; - s = config.SetResourceConfigMode(resource_mode); + /* gpu resource config */ + bool resource_enable_gpu = true; + s = config.SetGpuResourceConfigEnableGpu(std::to_string(resource_enable_gpu)); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigMode(str_val); + s = config.GetGpuResourceConfigEnableGpu(bool_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(str_val == resource_mode); + ASSERT_TRUE(bool_val == resource_enable_gpu); -#ifdef MILVUS_CPU_VERSION - std::vector search_resources = {"cpu"}; -#else - std::vector search_resources = {"cpu", "gpu0"}; -#endif - std::vector res_vec; - std::string res_str; - milvus::server::StringHelpFunctions::MergeStringWithDelimeter( - search_resources, milvus::server::CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_str); - s = config.SetResourceConfigSearchResources(res_str); +#ifdef MILVUS_GPU_VERSION + int64_t gpu_cache_capacity = 1; + s = config.SetGpuResourceConfigCacheCapacity(std::to_string(gpu_cache_capacity)); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigSearchResources(res_vec); + s = config.GetGpuResourceConfigCacheCapacity(int64_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int64_val == gpu_cache_capacity); + + float gpu_cache_threshold = 0.2; + s = config.SetGpuResourceConfigCacheThreshold(std::to_string(gpu_cache_threshold)); + ASSERT_TRUE(s.ok()); + s = config.GetGpuResourceConfigCacheThreshold(float_val); + ASSERT_TRUE(float_val == gpu_cache_threshold); + + std::vector search_resources = {"gpu0"}; + std::vector search_res_vec; + std::string search_res_str; + milvus::server::StringHelpFunctions::MergeStringWithDelimeter( + search_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, search_res_str); + s = config.SetGpuResourceConfigSearchResources(search_res_str); + ASSERT_TRUE(s.ok()); + s = config.GetGpuResourceConfigSearchResources(search_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < search_resources.size(); i++) { - ASSERT_TRUE(search_resources[i] == res_vec[i]); + ASSERT_TRUE(std::stoi(search_resources[i].substr(3)) == search_res_vec[i]); } -#ifdef MILVUS_CPU_VERSION - int32_t resource_index_build_device = milvus::server::CPU_DEVICE_ID; - s = config.SetResourceConfigIndexBuildDevice("cpu"); -#else - int32_t resource_index_build_device = 0; - s = config.SetResourceConfigIndexBuildDevice("gpu" + std::to_string(resource_index_build_device)); + std::vector build_index_resources = {"gpu0"}; + std::vector build_index_res_vec; + std::string build_index_res_str; + milvus::server::StringHelpFunctions::MergeStringWithDelimeter( + build_index_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, build_index_res_str); + s = config.SetGpuResourceConfigBuildIndexResources(build_index_res_str); + ASSERT_TRUE(s.ok()); + s = config.GetGpuResourceConfigBuildIndexResources(build_index_res_vec); + ASSERT_TRUE(s.ok()); + for (size_t i = 0; i < build_index_resources.size(); i++) { + ASSERT_TRUE(std::stoi(build_index_resources[i].substr(3)) == build_index_res_vec[i]); + } #endif - ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigIndexBuildDevice(int32_val); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == resource_index_build_device); } TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { @@ -381,18 +378,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetCacheConfigCpuCacheThreshold("1.0"); ASSERT_FALSE(s.ok()); -#ifdef MILVUS_GPU_VERSION - s = config.SetCacheConfigGpuCacheCapacity("a"); - ASSERT_FALSE(s.ok()); - s = config.SetCacheConfigGpuCacheCapacity("128"); - ASSERT_FALSE(s.ok()); - - s = config.SetCacheConfigGpuCacheThreshold("a"); - ASSERT_FALSE(s.ok()); - s = config.SetCacheConfigGpuCacheThreshold("1.0"); - ASSERT_FALSE(s.ok()); -#endif - s = config.SetCacheConfigCacheInsertData("N"); ASSERT_FALSE(s.ok()); @@ -408,20 +393,29 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetEngineConfigGpuSearchThreshold("-1"); ASSERT_FALSE(s.ok()); - /* resource config */ - s = config.SetResourceConfigMode("default"); + /* gpu resource config */ + s = config.SetGpuResourceConfigEnableGpu("ok"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigSearchResources("gpu10"); +#ifdef MILVUS_GPU_VERSION + s = config.SetGpuResourceConfigCacheCapacity("a"); + ASSERT_FALSE(s.ok()); + s = config.SetGpuResourceConfigCacheCapacity("128"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigSearchResources("cpu"); - ASSERT_TRUE(s.ok()); + s = config.SetGpuResourceConfigCacheThreshold("a"); + ASSERT_FALSE(s.ok()); + s = config.SetGpuResourceConfigCacheThreshold("1.0"); + ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigIndexBuildDevice("gup2"); + s = config.SetGpuResourceConfigSearchResources("gpu10"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigIndexBuildDevice("gpu16"); + + s = config.SetGpuResourceConfigBuildIndexResources("gup2"); ASSERT_FALSE(s.ok()); + s = config.SetGpuResourceConfigBuildIndexResources("gpu16"); + ASSERT_FALSE(s.ok()); +#endif } TEST_F(ConfigTest, SERVER_CONFIG_TEST) { From 9f063aa2893e6009d7b89d1fad75ca61c24dbba5 Mon Sep 17 00:00:00 2001 From: wxyu Date: Tue, 19 Nov 2019 16:53:04 +0800 Subject: [PATCH 27/82] Add a Fallback pass in optimizer close #409 add Fallback pass condition --- CHANGELOG.md | 1 + core/src/scheduler/SchedInst.h | 2 + core/src/scheduler/optimizer/FallbackPass.cpp | 43 +++++++++++++++++++ core/src/scheduler/optimizer/FallbackPass.h | 40 +++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 core/src/scheduler/optimizer/FallbackPass.cpp create mode 100644 core/src/scheduler/optimizer/FallbackPass.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 98bab925da..ced0633cdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#358 - Add more information in build.sh and install.md - \#255 - Add ivfsq8 test report detailed version - \#404 - Add virtual method Init() in Pass abstract class +- \#409 - Add a Fallback pass in optimizer ## Task diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index a3048069f9..ad02d32081 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -21,6 +21,7 @@ #include "JobMgr.h" #include "ResourceMgr.h" #include "Scheduler.h" +#include "optimizer/FallbackPass.h" #include "optimizer/HybridPass.h" #include "optimizer/LargeSQ8HPass.h" #include "optimizer/OnlyCPUPass.h" @@ -112,6 +113,7 @@ class OptimizerInst { pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared(has_cpu)); + pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); } } diff --git a/core/src/scheduler/optimizer/FallbackPass.cpp b/core/src/scheduler/optimizer/FallbackPass.cpp new file mode 100644 index 0000000000..2e275ede4b --- /dev/null +++ b/core/src/scheduler/optimizer/FallbackPass.cpp @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FallbackPass.h" +#include "scheduler/SchedInst.h" +#include "scheduler/tasklabel/SpecResLabel.h" + +namespace milvus { +namespace scheduler { + +void +FallbackPass::Init() { +} + +bool +FallbackPass::Run(const TaskPtr& task) { + auto task_type = task->Type(); + if (task_type != TaskType::SearchTask && task_type != TaskType::BuildIndexTask) { + return false; + } + // NEVER be empty + auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0]; + auto label = std::make_shared(cpu); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/FallbackPass.h b/core/src/scheduler/optimizer/FallbackPass.h new file mode 100644 index 0000000000..728740d53a --- /dev/null +++ b/core/src/scheduler/optimizer/FallbackPass.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class FallbackPass : public Pass { + public: + FallbackPass() = default; + + public: + void + Init() override; + + bool + Run(const TaskPtr& task) override; +}; + +} // namespace scheduler +} // namespace milvus From 0011765b2db28ca0550a5f1ae31c1299a3cc7799 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Tue, 19 Nov 2019 17:30:29 +0800 Subject: [PATCH 28/82] Change ivfsq8 test report detailed version #255 --- .../milvus_ivfsq8_test_report_detailed_version.md | 8 ++++---- .../milvus_ivfsq8_test_report_detailed_version_cn.md | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md index dc1bbb3fd0..7bea764df0 100644 --- a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md @@ -28,13 +28,13 @@ Memory: 503GB Docker version: 18.09 -Nvidia Driver version: 430.34 +NVIDIA Driver version: 430.34 Milvus version: 0.5.3 SDK interface: Python 3.6.8 -Pymilvus version: 0.2.5 +pymilvus version: 0.2.5 @@ -66,7 +66,7 @@ For details on this dataset, you can check : http://corpus-texmex.irisa.fr/ . > Note: In the query test of recall, we will test the following parameters with different values: > - > nq - grouped by: [1, 5, 10, 200, 400, 600, 800, 1000], + > nq - grouped by: [10, 200, 400, 600, 800, 1000], > > topk - grouped by: [1, 10, 100] @@ -93,7 +93,7 @@ Milvus configuration - gpu_cache_capacity: 6 - use_blas_threshold: 1100 -You can check the definition of Milvus configuration on https://milvus.io/docs/en/reference/milvus_config/. +The definitions of Milvus configuration are on https://milvus.io/docs/en/reference/milvus_config/. Test method diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md index bf43d2e0e8..a6e5e75ea4 100644 --- a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md @@ -2,7 +2,7 @@ ## 概述 -本文描述了ivfsq8索引在milvus单机部署方式下的测试报告。 +本文描述了ivfsq8索引在milvus单机部署方式下的测试结果。 @@ -28,13 +28,13 @@ GPU1: GeForce GTX 1080 Docker版本: 18.09 -Nvidia Driver版本: 430.34 +NVIDIA Driver版本: 430.34 Milvus版本: 0.5.3 SDK接口: Python 3.6.8 -Pymilvus版本: 0.2.5 +pymilvus版本: 0.2.5 @@ -66,7 +66,7 @@ Pymilvus版本: 0.2.5 > 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果： > - > 被查询向量的数量nq将按照 [1, 5, 10, 200, 400, 600, 800, 1000]的数量分组， + > 被查询向量的数量nq将按照 [10, 200, 400, 600, 800, 1000]的数量分组， > > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。 @@ -93,7 +93,7 @@ Milvus设置： - gpu_cache_capacity: 6 - use_blas_threshold: 1100 -你可以在 https://milvus.io/docs/en/reference/milvus_config/上查询Milvus设置的详细定义。 +Milvus设置的详细定义可以参考 https://milvus.io/docs/en/reference/milvus_config/ 。 测试方法 From 99deaf5c503b20b6af135549bc94a2b28c7d9da1 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Tue, 19 Nov 2019 17:37:13 +0800 Subject: [PATCH 29/82] modify shards for v0.5.3 --- shards/mishards/connections.py | 10 +++ shards/mishards/service_handler.py | 107 +++++++++++++++++++++-------- 2 files changed, 89 insertions(+), 28 deletions(-) diff --git a/shards/mishards/connections.py b/shards/mishards/connections.py index 618690a099..50e214ec9a 100644 --- a/shards/mishards/connections.py +++ b/shards/mishards/connections.py @@ -2,6 +2,7 @@ import logging import threading from functools import wraps from milvus import Milvus +from milvus.client.hooks import BaseaSearchHook from mishards import (settings, exceptions) from utils import singleton @@ -9,6 +10,12 @@ from utils import singleton logger = logging.getLogger(__name__) +class Searchook(BaseaSearchHook): + + def on_response(self, *args, **kwargs): + return True + + class Connection: def __init__(self, name, uri, max_retry=1, error_handlers=None, **kwargs): self.name = name @@ -18,6 +25,9 @@ class Connection: self.conn = Milvus() self.error_handlers = [] if not error_handlers else error_handlers self.on_retry_func = kwargs.get('on_retry_func', None) + + # define search hook + self.conn._set_hook(search_in_file=Searchook()) # self._connect() def __str__(self): diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 2f19152ae6..620f6213de 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -29,39 +29,88 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): self.router = router self.max_workers = max_workers + def _reduce(self, source_ids, ids, source_diss, diss, k, reverse): + if source_diss[k - 1] <= diss[0]: + return source_ids, source_diss + if diss[k - 1] <= source_diss[0]: + return ids, diss + + diss_t = enumerate(source_diss.extend(diss)) + diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] + diss_m_out = [id_ for _, id_ in diss_m_rst] + + id_t = source_ids.extend(ids) + id_m_out = [id_t[i] for i, _ in diss_m_rst] + + return id_m_out, diss_m_out + def _do_merge(self, files_n_topk_results, topk, reverse=False, **kwargs): status = status_pb2.Status(error_code=status_pb2.SUCCESS, reason="Success") if not files_n_topk_results: return status, [] - request_results = defaultdict(list) + # request_results = defaultdict(list) + # row_num = files_n_topk_results[0].row_num + merge_id_results = [] + merge_dis_results = [] calc_time = time.time() for files_collection in files_n_topk_results: if isinstance(files_collection, tuple): status, _ = files_collection return status, [] - for request_pos, each_request_results in enumerate( - files_collection.topk_query_result): - request_results[request_pos].extend( - each_request_results.query_result_arrays) - request_results[request_pos] = sorted( - request_results[request_pos], - key=lambda x: x.distance, - reverse=reverse)[:topk] + + row_num = files_collection.row_num + ids = files_collection.ids + diss = files_collection.distances # distance collections + batch_len = len(ids) // row_num + + for row_index in range(row_num): + id_batch = ids[row_index * batch_len: (row_index + 1) * batch_len] + dis_batch = diss[row_index * batch_len: (row_index + 1) * batch_len] + + if len(merge_id_results) < row_index: + raise ValueError("merge error") + elif len(merge_id_results) == row_index: + # TODO: may bug here + merge_id_results.append(id_batch) + merge_dis_results.append(dis_batch) + else: + merge_id_results[row_index].extend(ids[row_index * batch_len, (row_index + 1) * batch_len]) + merge_dis_results[row_index].extend(diss[row_index * batch_len, (row_index + 1) * batch_len]) + # _reduce(_ids, _diss, k, reverse) + merge_id_results[row_index], merge_dis_results[row_index] = \ + self._reduce(merge_id_results[row_index], id_batch, + merge_dis_results[row_index], dis_batch, + batch_len, + reverse) + + # for request_pos, each_request_results in enumerate( + # files_collection.topk_query_result): + # request_results[request_pos].extend( + # each_request_results.query_result_arrays) + # request_results[request_pos] = sorted( + # request_results[request_pos], + # key=lambda x: x.distance, + # reverse=reverse)[:topk] calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - results = sorted(request_results.items()) - topk_query_result = [] + # results = sorted(request_results.items()) + id_mrege_list = [] + dis_mrege_list = [] - for result in results: - query_result = TopKQueryResult(query_result_arrays=result[1]) - topk_query_result.append(query_result) + for id_results, dis_results in zip(merge_id_results, merge_dis_results): + id_mrege_list.extend(id_results) + dis_mrege_list.extend(dis_results) - return status, topk_query_result + # for result in results: + # query_result = TopKQueryResult(query_result_arrays=result[1]) + # topk_query_result.append(query_result) + + return status, id_mrege_list, dis_mrege_list def _do_query(self, context, @@ -109,8 +158,8 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): file_ids=query_params['file_ids'], query_records=vectors, top_k=topk, - nprobe=nprobe, - lazy_=True) + nprobe=nprobe + ) end = time.time() logger.info('search_vectors_in_files takes: {}'.format(end - start)) @@ -241,7 +290,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): logger.info('Search {}: topk={} nprobe={}'.format( table_name, topk, nprobe)) - metadata = {'resp_class': milvus_pb2.TopKQueryResultList} + metadata = {'resp_class': milvus_pb2.TopKQueryResult} if nprobe > self.MAX_NPROBE or nprobe <= 0: raise exceptions.InvalidArgumentError( @@ -275,22 +324,24 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): query_range_array.append( Range(query_range.start_value, query_range.end_value)) - status, results = self._do_query(context, - table_name, - table_meta, - query_record_array, - topk, - nprobe, - query_range_array, - metadata=metadata) + status, id_results, dis_results = self._do_query(context, + table_name, + table_meta, + query_record_array, + topk, + nprobe, + query_range_array, + metadata=metadata) now = time.time() logger.info('SearchVector takes: {}'.format(now - start)) - topk_result_list = milvus_pb2.TopKQueryResultList( + topk_result_list = milvus_pb2.TopKQueryResult( status=status_pb2.Status(error_code=status.error_code, reason=status.reason), - topk_query_result=results) + row_num=len(query_record_array), + ids=id_results, + distances=dis_results) return topk_result_list @mark_grpc_method From c451f8cbf2b66c45759f7ca7bba15212343e2a91 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Tue, 19 Nov 2019 19:40:17 +0800 Subject: [PATCH 30/82] clange format and add FallbackPass --- core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h | 1 - core/src/scheduler/SchedInst.h | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h index 4bb4092b75..e064b6f08c 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h @@ -47,7 +47,6 @@ class IVF : public VectorIndex, public FaissBaseIndex { void set_index_model(IndexModelPtr model) override; - void Add(const DatasetPtr& dataset, const Config& config) override; diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 48eaffc0a5..44fea1b6d3 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -104,14 +104,13 @@ class OptimizerInst { pass_list.push_back(std::make_shared()); #ifdef MILVUS_CPU_VERSION pass_list.push_back(std::make_shared()); -<<<<<<< HEAD #else server::Config& config = server::Config::GetInstance(); std::vector build_resources; config.GetGpuResourceConfigBuildIndexResources(build_resources); pass_list.push_back(std::make_shared(build_resources)); #endif -// pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); } } From fdb7decdd3debbeae3d74cd7fd5ec608b319ca19 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Tue, 19 Nov 2019 20:30:43 +0800 Subject: [PATCH 31/82] Remove DefaultLabel --- core/src/scheduler/JobMgr.cpp | 8 +-- core/src/scheduler/JobMgr.h | 5 +- core/src/scheduler/Scheduler.cpp | 4 -- core/src/scheduler/TaskCreator.cpp | 9 +-- core/src/scheduler/Utils.cpp | 1 - core/src/scheduler/action/Action.h | 4 -- .../scheduler/action/PushTaskToNeighbour.cpp | 32 --------- core/src/scheduler/tasklabel/DefaultLabel.h | 36 ---------- core/src/scheduler/tasklabel/TaskLabel.h | 1 - core/unittest/scheduler/CMakeLists.txt | 1 - core/unittest/scheduler/test_normal.cpp | 72 ------------------- core/unittest/scheduler/test_resource.cpp | 18 +++-- core/unittest/scheduler/test_resource_mgr.cpp | 4 +- core/unittest/scheduler/test_scheduler.cpp | 41 ----------- core/unittest/scheduler/test_tasktable.cpp | 9 +-- 15 files changed, 26 insertions(+), 219 deletions(-) delete mode 100644 core/src/scheduler/tasklabel/DefaultLabel.h delete mode 100644 core/unittest/scheduler/test_normal.cpp diff --git a/core/src/scheduler/JobMgr.cpp b/core/src/scheduler/JobMgr.cpp index 8dd095a3fa..76c07fe459 100644 --- a/core/src/scheduler/JobMgr.cpp +++ b/core/src/scheduler/JobMgr.cpp @@ -85,7 +85,7 @@ JobMgr::worker_function() { } for (auto& task : tasks) { - calculate_path(task); + calculate_path(res_mgr_, task); } // disk resources NEVER be empty. @@ -103,7 +103,7 @@ JobMgr::build_task(const JobPtr& job) { } void -JobMgr::calculate_path(const TaskPtr& task) { +JobMgr::calculate_path(const ResourceMgrPtr& res_mgr, const TaskPtr& task) { if (task->type_ != TaskType::SearchTask && task->type_ != TaskType::BuildIndexTask) { return; } @@ -114,9 +114,9 @@ JobMgr::calculate_path(const TaskPtr& task) { std::vector path; auto spec_label = std::static_pointer_cast(task->label()); - auto src = res_mgr_->GetDiskResources()[0]; + auto src = res_mgr->GetDiskResources()[0]; auto dest = spec_label->resource(); - ShortestPath(src.lock(), dest.lock(), res_mgr_, path); + ShortestPath(src.lock(), dest.lock(), res_mgr, path); task->path() = Path(path, path.size() - 1); } diff --git a/core/src/scheduler/JobMgr.h b/core/src/scheduler/JobMgr.h index fbd6c0ee45..af072614b5 100644 --- a/core/src/scheduler/JobMgr.h +++ b/core/src/scheduler/JobMgr.h @@ -59,8 +59,9 @@ class JobMgr : public interface::dumpable { static std::vector build_task(const JobPtr& job); - void - calculate_path(const TaskPtr& task); + public: + static void + calculate_path(const ResourceMgrPtr& res_mgr, const TaskPtr& task); private: bool running_ = false; diff --git a/core/src/scheduler/Scheduler.cpp b/core/src/scheduler/Scheduler.cpp index 8d2d4406f8..68d7457aa9 100644 --- a/core/src/scheduler/Scheduler.cpp +++ b/core/src/scheduler/Scheduler.cpp @@ -108,10 +108,6 @@ Scheduler::OnLoadCompleted(const EventPtr& event) { auto task_table_type = load_completed_event->task_table_item_->task->label()->Type(); switch (task_table_type) { - case TaskLabelType::DEFAULT: { - Action::DefaultLabelTaskScheduler(res_mgr_, resource, load_completed_event); - break; - } case TaskLabelType::SPECIFIED_RESOURCE: { Action::SpecifiedResourceLabelTaskScheduler(res_mgr_, resource, load_completed_event); break; diff --git a/core/src/scheduler/TaskCreator.cpp b/core/src/scheduler/TaskCreator.cpp index 3b7f6fd82e..30b76cc5bb 100644 --- a/core/src/scheduler/TaskCreator.cpp +++ b/core/src/scheduler/TaskCreator.cpp @@ -18,7 +18,6 @@ #include "scheduler/TaskCreator.h" #include "SchedInst.h" #include "tasklabel/BroadcastLabel.h" -#include "tasklabel/DefaultLabel.h" #include "tasklabel/SpecResLabel.h" namespace milvus { @@ -47,8 +46,7 @@ std::vector TaskCreator::Create(const SearchJobPtr& job) { std::vector tasks; for (auto& index_file : job->index_files()) { - auto label = std::make_shared(); - auto task = std::make_shared(index_file.second, label); + auto task = std::make_shared(index_file.second, nullptr); task->job_ = job; tasks.emplace_back(task); } @@ -70,11 +68,8 @@ TaskCreator::Create(const DeleteJobPtr& job) { std::vector TaskCreator::Create(const BuildIndexJobPtr& job) { std::vector tasks; - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); - for (auto& to_index_file : job->to_index_files()) { - auto label = std::make_shared(std::weak_ptr(res_ptr)); - auto task = std::make_shared(to_index_file.second, label); + auto task = std::make_shared(to_index_file.second, nullptr); task->job_ = job; tasks.emplace_back(task); } diff --git a/core/src/scheduler/Utils.cpp b/core/src/scheduler/Utils.cpp index 1be04ba80b..527b9adf03 100644 --- a/core/src/scheduler/Utils.cpp +++ b/core/src/scheduler/Utils.cpp @@ -16,7 +16,6 @@ // under the License. #include "scheduler/Utils.h" -#include "server/Config.h" #ifdef MILVUS_GPU_VERSION #include diff --git a/core/src/scheduler/action/Action.h b/core/src/scheduler/action/Action.h index f5f828cbf6..7391d287a8 100644 --- a/core/src/scheduler/action/Action.h +++ b/core/src/scheduler/action/Action.h @@ -36,10 +36,6 @@ class Action { static void PushTaskToResource(TaskTableItemPtr task_item, const ResourcePtr& dest); - static void - DefaultLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, - std::shared_ptr event); - static void SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, std::shared_ptr event); diff --git a/core/src/scheduler/action/PushTaskToNeighbour.cpp b/core/src/scheduler/action/PushTaskToNeighbour.cpp index 9aed678937..f49f1d871f 100644 --- a/core/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/core/src/scheduler/action/PushTaskToNeighbour.cpp @@ -101,38 +101,6 @@ Action::PushTaskToResource(TaskTableItemPtr task_item, const ResourcePtr& dest) dest->task_table().Put(task_item->task, task_item); } -void -Action::DefaultLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, - std::shared_ptr event) { - if (not resource->HasExecutor() && event->task_table_item_->Move()) { - auto task_item = event->task_table_item_; - auto task = event->task_table_item_->task; - auto search_task = std::static_pointer_cast(task); - bool moved = false; - - // to support test task, REFACTOR - if (resource->type() == ResourceType::CPU) { - if (auto index_engine = search_task->index_engine_) { - auto location = index_engine->GetLocation(); - - for (auto i = 0; i < res_mgr->GetNumGpuResource(); ++i) { - auto index = milvus::cache::GpuCacheMgr::GetInstance(i)->GetIndex(location); - if (index != nullptr) { - moved = true; - auto dest_resource = res_mgr->GetResource(ResourceType::GPU, i); - PushTaskToResource(event->task_table_item_, dest_resource); - break; - } - } - } - } - - if (not moved) { - PushTaskToNeighbourRandomly(task_item, resource); - } - } -} - void Action::SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, std::shared_ptr event) { diff --git a/core/src/scheduler/tasklabel/DefaultLabel.h b/core/src/scheduler/tasklabel/DefaultLabel.h deleted file mode 100644 index c215743575..0000000000 --- a/core/src/scheduler/tasklabel/DefaultLabel.h +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "TaskLabel.h" - -#include - -namespace milvus { -namespace scheduler { - -class DefaultLabel : public TaskLabel { - public: - DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) { - } -}; - -using DefaultLabelPtr = std::shared_ptr; - -} // namespace scheduler -} // namespace milvus diff --git a/core/src/scheduler/tasklabel/TaskLabel.h b/core/src/scheduler/tasklabel/TaskLabel.h index d35ce409ff..33e6eb6e57 100644 --- a/core/src/scheduler/tasklabel/TaskLabel.h +++ b/core/src/scheduler/tasklabel/TaskLabel.h @@ -23,7 +23,6 @@ namespace milvus { namespace scheduler { enum class TaskLabelType { - DEFAULT, // means can be executed in any resource SPECIFIED_RESOURCE, // means must executing in special resource BROADCAST, // means all enable-executor resource must execute task }; diff --git a/core/unittest/scheduler/CMakeLists.txt b/core/unittest/scheduler/CMakeLists.txt index 0148441c2d..878f0f23a9 100644 --- a/core/unittest/scheduler/CMakeLists.txt +++ b/core/unittest/scheduler/CMakeLists.txt @@ -21,7 +21,6 @@ set(test_files ${CMAKE_CURRENT_SOURCE_DIR}/test_algorithm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_node.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/test_normal.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_resource.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_resource_factory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_resource_mgr.cpp diff --git a/core/unittest/scheduler/test_normal.cpp b/core/unittest/scheduler/test_normal.cpp deleted file mode 100644 index 20cd39d0bf..0000000000 --- a/core/unittest/scheduler/test_normal.cpp +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include "scheduler/ResourceFactory.h" -#include "scheduler/ResourceMgr.h" -#include "scheduler/SchedInst.h" -#include "scheduler/Scheduler.h" -#include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" -#include "utils/Log.h" - -namespace { - -namespace ms = milvus::scheduler; - -} // namespace - -TEST(NormalTest, INST_TEST) { - // ResourceMgr only compose resources, provide unified event - auto res_mgr = ms::ResMgrInst::GetInstance(); - - res_mgr->Add(ms::ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(ms::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - - auto IO = ms::Connection("IO", 500.0); - res_mgr->Connect("disk", "cpu", IO); - - auto scheduler = ms::SchedInst::GetInstance(); - - res_mgr->Start(); - scheduler->Start(); - - const uint64_t NUM_TASK = 2; - std::vector> tasks; - ms::TableFileSchemaPtr dummy = nullptr; - - auto disks = res_mgr->GetDiskResources(); - ASSERT_FALSE(disks.empty()); - if (auto observe = disks[0].lock()) { - for (uint64_t i = 0; i < NUM_TASK; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); - task->label() = std::make_shared(); - tasks.push_back(task); - observe->task_table().Put(task); - } - } - - for (auto& task : tasks) { - task->Wait(); - ASSERT_EQ(task->load_count_, 1); - ASSERT_EQ(task->exec_count_, 1); - } - - scheduler->Stop(); - res_mgr->Stop(); -} diff --git a/core/unittest/scheduler/test_resource.cpp b/core/unittest/scheduler/test_resource.cpp index 6fe85ec558..101dcd3976 100644 --- a/core/unittest/scheduler/test_resource.cpp +++ b/core/unittest/scheduler/test_resource.cpp @@ -24,7 +24,7 @@ #include "scheduler/resource/TestResource.h" #include "scheduler/task/Task.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" +#include "scheduler/tasklabel/SpecResLabel.h" namespace milvus { namespace scheduler { @@ -182,8 +182,10 @@ TEST_F(ResourceAdvanceTest, DISK_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(disk_resource_); auto task = std::make_shared(dummy, label); + std::vector path{disk_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); disk_resource_->task_table().Put(task); } @@ -208,8 +210,10 @@ TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(cpu_resource_); auto task = std::make_shared(dummy, label); + std::vector path{cpu_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); cpu_resource_->task_table().Put(task); } @@ -234,8 +238,10 @@ TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(gpu_resource_); auto task = std::make_shared(dummy, label); + std::vector path{gpu_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); gpu_resource_->task_table().Put(task); } @@ -260,8 +266,10 @@ TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(test_resource_); auto task = std::make_shared(dummy, label); + std::vector path{test_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); test_resource_->task_table().Put(task); } diff --git a/core/unittest/scheduler/test_resource_mgr.cpp b/core/unittest/scheduler/test_resource_mgr.cpp index b9127060bd..d6495971c4 100644 --- a/core/unittest/scheduler/test_resource_mgr.cpp +++ b/core/unittest/scheduler/test_resource_mgr.cpp @@ -22,7 +22,6 @@ #include "scheduler/resource/GpuResource.h" #include "scheduler/resource/TestResource.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" namespace milvus { namespace scheduler { @@ -187,8 +186,7 @@ TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) { auto callback = [&](EventPtr event) { flag = true; }; mgr1_->RegisterSubscriber(callback); TableFileSchemaPtr dummy = nullptr; - auto label = std::make_shared(); - disk_res->task_table().Put(std::make_shared(dummy, label)); + disk_res->task_table().Put(std::make_shared(dummy, nullptr)); sleep(1); ASSERT_TRUE(flag); } diff --git a/core/unittest/scheduler/test_scheduler.cpp b/core/unittest/scheduler/test_scheduler.cpp index 22fb9be723..72538113c3 100644 --- a/core/unittest/scheduler/test_scheduler.cpp +++ b/core/unittest/scheduler/test_scheduler.cpp @@ -23,7 +23,6 @@ #include "scheduler/Scheduler.h" #include "scheduler/resource/Resource.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" #include "scheduler/tasklabel/SpecResLabel.h" #include "utils/Error.h" #include "wrapper/VecIndex.h" @@ -150,46 +149,6 @@ insert_dummy_index_into_gpu_cache(uint64_t device_id) { cache::GpuCacheMgr::GetInstance(device_id)->InsertItem("location", obj); } -TEST_F(SchedulerTest, ON_LOAD_COMPLETED) { - const uint64_t NUM = 10; - std::vector> tasks; - TableFileSchemaPtr dummy = std::make_shared(); - dummy->location_ = "location"; - - insert_dummy_index_into_gpu_cache(1); - - for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); - task->label() = std::make_shared(); - tasks.push_back(task); - cpu_resource_.lock()->task_table().Put(task); - } - - sleep(3); - ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().size(), NUM); -} - -TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) { - const uint64_t NUM = 10; - std::vector> tasks; - TableFileSchemaPtr dummy1 = std::make_shared(); - dummy1->location_ = "location"; - - tasks.clear(); - - for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy1, label); - task->label() = std::make_shared(); - tasks.push_back(task); - cpu_resource_.lock()->task_table().Put(task); - } - - sleep(3); - // ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM); -} - class SchedulerTest2 : public testing::Test { protected: void diff --git a/core/unittest/scheduler/test_tasktable.cpp b/core/unittest/scheduler/test_tasktable.cpp index 28a2e29c98..3fa1beabeb 100644 --- a/core/unittest/scheduler/test_tasktable.cpp +++ b/core/unittest/scheduler/test_tasktable.cpp @@ -18,7 +18,6 @@ #include #include "scheduler/TaskTable.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" /************ TaskTableBaseTest ************/ @@ -162,9 +161,8 @@ class TaskTableBaseTest : public ::testing::Test { SetUp() override { milvus::scheduler::TableFileSchemaPtr dummy = nullptr; invalid_task_ = nullptr; - auto label = std::make_shared(); - task1_ = std::make_shared(dummy, label); - task2_ = std::make_shared(dummy, label); + task1_ = std::make_shared(dummy, nullptr); + task2_ = std::make_shared(dummy, nullptr); } milvus::scheduler::TaskPtr invalid_task_; @@ -320,8 +318,7 @@ class TaskTableAdvanceTest : public ::testing::Test { SetUp() override { milvus::scheduler::TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < 8; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); + auto task = std::make_shared(dummy, nullptr); table1_.Put(task); } From 67605968b8977ddb9e29c7f0a2ea8d49a3e4f703 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Tue, 19 Nov 2019 20:36:08 +0800 Subject: [PATCH 32/82] finish results reduce in mishards --- shards/mishards/service_handler.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 620f6213de..640ae61ba8 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -34,13 +34,14 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): return source_ids, source_diss if diss[k - 1] <= source_diss[0]: return ids, diss - - diss_t = enumerate(source_diss.extend(diss)) + + source_diss.extend(diss) + diss_t = enumerate(source_diss) diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] diss_m_out = [id_ for _, id_ in diss_m_rst] - id_t = source_ids.extend(ids) - id_m_out = [id_t[i] for i, _ in diss_m_rst] + source_ids.extend(ids) + id_m_out = [source_ids[i] for i, _ in diss_m_rst] return id_m_out, diss_m_out @@ -50,8 +51,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): if not files_n_topk_results: return status, [] - # request_results = defaultdict(list) - # row_num = files_n_topk_results[0].row_num merge_id_results = [] merge_dis_results = [] @@ -64,6 +63,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): row_num = files_collection.row_num ids = files_collection.ids diss = files_collection.distances # distance collections + # TODO: batch_len is equal to topk batch_len = len(ids) // row_num for row_index in range(row_num): @@ -77,28 +77,16 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): merge_id_results.append(id_batch) merge_dis_results.append(dis_batch) else: - merge_id_results[row_index].extend(ids[row_index * batch_len, (row_index + 1) * batch_len]) - merge_dis_results[row_index].extend(diss[row_index * batch_len, (row_index + 1) * batch_len]) - # _reduce(_ids, _diss, k, reverse) merge_id_results[row_index], merge_dis_results[row_index] = \ self._reduce(merge_id_results[row_index], id_batch, merge_dis_results[row_index], dis_batch, batch_len, reverse) - # for request_pos, each_request_results in enumerate( - # files_collection.topk_query_result): - # request_results[request_pos].extend( - # each_request_results.query_result_arrays) - # request_results[request_pos] = sorted( - # request_results[request_pos], - # key=lambda x: x.distance, - # reverse=reverse)[:topk] calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - # results = sorted(request_results.items()) id_mrege_list = [] dis_mrege_list = [] @@ -106,10 +94,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): id_mrege_list.extend(id_results) dis_mrege_list.extend(dis_results) - # for result in results: - # query_result = TopKQueryResult(query_result_arrays=result[1]) - # topk_query_result.append(query_result) - return status, id_mrege_list, dis_mrege_list def _do_query(self, From a13677b65f252b9c4025c4c2e3a25dfb8e6e112c Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Tue, 19 Nov 2019 21:10:43 +0800 Subject: [PATCH 33/82] Refactor optimizer --- core/src/scheduler/SchedInst.h | 22 +++--- .../scheduler/optimizer/BuildIndexPass.cpp | 9 ++- core/src/scheduler/optimizer/BuildIndexPass.h | 2 +- .../src/scheduler/optimizer/FaissFlatPass.cpp | 69 +++++++++++++++++++ .../{LargeSQ8HPass.h => FaissFlatPass.h} | 6 +- .../{OnlyCPUPass.cpp => FaissIVFFlatPass.cpp} | 41 ++++++++--- .../{HybridPass.h => FaissIVFFlatPass.h} | 12 +++- ...LargeSQ8HPass.cpp => FaissIVFSQ8HPass.cpp} | 40 +++-------- .../scheduler/optimizer/FaissIVFSQ8HPass.h | 56 +++++++++++++++ .../scheduler/optimizer/FaissIVFSQ8Pass.cpp | 69 +++++++++++++++++++ .../{OnlyCPUPass.h => FaissIVFSQ8Pass.h} | 12 +++- core/src/scheduler/optimizer/HybridPass.cpp | 47 ------------- 12 files changed, 273 insertions(+), 112 deletions(-) create mode 100644 core/src/scheduler/optimizer/FaissFlatPass.cpp rename core/src/scheduler/optimizer/{LargeSQ8HPass.h => FaissFlatPass.h} (91%) rename core/src/scheduler/optimizer/{OnlyCPUPass.cpp => FaissIVFFlatPass.cpp} (56%) rename core/src/scheduler/optimizer/{HybridPass.h => FaissIVFFlatPass.h} (81%) rename core/src/scheduler/optimizer/{LargeSQ8HPass.cpp => FaissIVFSQ8HPass.cpp} (61%) create mode 100644 core/src/scheduler/optimizer/FaissIVFSQ8HPass.h create mode 100644 core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp rename core/src/scheduler/optimizer/{OnlyCPUPass.h => FaissIVFSQ8Pass.h} (81%) delete mode 100644 core/src/scheduler/optimizer/HybridPass.cpp diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 44fea1b6d3..912f5edf8d 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -24,9 +24,10 @@ #include "Utils.h" #include "optimizer/BuildIndexPass.h" #include "optimizer/FallbackPass.h" -#include "optimizer/HybridPass.h" -#include "optimizer/LargeSQ8HPass.h" -#include "optimizer/OnlyCPUPass.h" +#include "optimizer/FaissFlatPass.h" +#include "optimizer/FaissIVFFlatPass.h" +#include "optimizer/FaissIVFSQ8Pass.h" +#include "optimizer/FaissIVFSQ8HPass.h" #include "optimizer/Optimizer.h" #include "server/Config.h" @@ -100,15 +101,12 @@ class OptimizerInst { std::lock_guard lock(mutex_); if (instance == nullptr) { std::vector pass_list; - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); -#ifdef MILVUS_CPU_VERSION - pass_list.push_back(std::make_shared()); -#else - server::Config& config = server::Config::GetInstance(); - std::vector build_resources; - config.GetGpuResourceConfigBuildIndexResources(build_resources); - pass_list.push_back(std::make_shared(build_resources)); +#ifdef MILVUS_GPU_VERSION + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); #endif pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index 4e7d5222e2..cd1b7550d8 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -23,11 +23,14 @@ namespace milvus { namespace scheduler { -BuildIndexPass::BuildIndexPass(std::vector& build_gpu_ids) : build_gpu_ids_(build_gpu_ids) { -} - void BuildIndexPass::Init() { + server::Config& config = server::Config::GetInstance(); + std::vector build_resources; + Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); + if (!s.ok()) { + throw; + } } bool diff --git a/core/src/scheduler/optimizer/BuildIndexPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h index a3eb426527..4f7117fc4e 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.h +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -34,7 +34,7 @@ namespace scheduler { class BuildIndexPass : public Pass { public: - explicit BuildIndexPass(std::vector& build_gpu_id); + BuildIndexPass() = default; public: void diff --git a/core/src/scheduler/optimizer/FaissFlatPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp new file mode 100644 index 0000000000..61ca1b9ec9 --- /dev/null +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FaissFlatPass.h" +#include "cache/GpuCacheMgr.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" + +namespace milvus { +namespace scheduler { + +void +FaissFlatPass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } +} + +bool +FaissFlatPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { + return false; + } + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IDMAP) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/FaissFlatPass.h similarity index 91% rename from core/src/scheduler/optimizer/LargeSQ8HPass.h rename to core/src/scheduler/optimizer/FaissFlatPass.h index c69fb12c64..60adf62a07 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.h +++ b/core/src/scheduler/optimizer/FaissFlatPass.h @@ -33,9 +33,9 @@ namespace milvus { namespace scheduler { -class LargeSQ8HPass : public Pass { +class FaissFlatPass : public Pass { public: - LargeSQ8HPass() = default; + FaissFlatPass() = default; public: void @@ -50,7 +50,7 @@ class LargeSQ8HPass : public Pass { std::vector gpus; }; -using LargeSQ8HPassPtr = std::shared_ptr; +using FaissFlatPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp similarity index 56% rename from core/src/scheduler/optimizer/OnlyCPUPass.cpp rename to core/src/scheduler/optimizer/FaissIVFFlatPass.cpp index 8cd40092fb..1f1efb374b 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -15,31 +15,52 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/OnlyCPUPass.h" +#include "scheduler/optimizer/FaissIVFFlatPass.h" +#include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" #include "scheduler/task/SearchTask.h" #include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" namespace milvus { namespace scheduler { void -OnlyCPUPass::Init() { +FaissIVFFlatPass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } } bool -OnlyCPUPass::Run(const TaskPtr& task) { - if (task->Type() != TaskType::SearchTask) - return false; - auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8 && - search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { +FaissIVFFlatPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { return false; } - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); - auto label = std::make_shared(std::weak_ptr(res_ptr)); + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); task->label() = label; return true; } diff --git a/core/src/scheduler/optimizer/HybridPass.h b/core/src/scheduler/optimizer/FaissIVFFlatPass.h similarity index 81% rename from core/src/scheduler/optimizer/HybridPass.h rename to core/src/scheduler/optimizer/FaissIVFFlatPass.h index f84a0884f2..9264db32a8 100644 --- a/core/src/scheduler/optimizer/HybridPass.h +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -32,9 +33,9 @@ namespace milvus { namespace scheduler { -class HybridPass : public Pass { +class FaissIVFFlatPass : public Pass { public: - HybridPass() = default; + FaissIVFFlatPass() = default; public: void @@ -42,9 +43,14 @@ class HybridPass : public Pass { bool Run(const TaskPtr& task) override; + + private: + int32_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; }; -using HybridPassPtr = std::shared_ptr; +using FaissIVFFlatPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp similarity index 61% rename from core/src/scheduler/optimizer/LargeSQ8HPass.cpp rename to core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index dd58feb512..b63a0f7490 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/LargeSQ8HPass.h" +#include "scheduler/optimizer/FaissIVFSQ8HPass.h" #include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" @@ -28,7 +28,7 @@ namespace milvus { namespace scheduler { void -LargeSQ8HPass::Init() { +FaissIVFSQ8HPass::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { @@ -38,7 +38,7 @@ LargeSQ8HPass::Init() { } bool -LargeSQ8HPass::Run(const TaskPtr& task) { +FaissIVFSQ8HPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::SearchTask) { return false; } @@ -49,36 +49,16 @@ LargeSQ8HPass::Run(const TaskPtr& task) { } auto search_job = std::static_pointer_cast(search_task->job_.lock()); - - // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu - + ResourcePtr res_ptr; if (search_job->nq() < threshold_) { - return false; + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } - - // std::vector all_free_mem; - // for (auto& gpu : gpus) { - // auto cache = cache::GpuCacheMgr::GetInstance(gpu); - // auto free_mem = cache->CacheCapacity() - cache->CacheUsage(); - // all_free_mem.push_back(free_mem); - // } - // - // auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end()); - // auto best_index = std::distance(all_free_mem.begin(), max_e); - // auto best_device_id = gpus[best_index]; - auto best_device_id = count_ % gpus.size(); - count_++; - - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); - if (not res_ptr) { - SERVER_LOG_ERROR << "GpuResource " << best_device_id << " invalid."; - // TODO: throw critical error and exit - return false; - } - - auto label = std::make_shared(std::weak_ptr(res_ptr)); + auto label = std::make_shared(res_ptr); task->label() = label; - return true; } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h new file mode 100644 index 0000000000..fa830ff08a --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class FaissIVFSQ8HPass : public Pass { + public: + FaissIVFSQ8HPass() = default; + + public: + void + Init() override; + + bool + Run(const TaskPtr& task) override; + + private: + int32_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; +}; + +using FaissIVFSQ8HPassPtr = std::shared_ptr; + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp new file mode 100644 index 0000000000..30dd306b3b --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FaissIVFSQ8Pass.h" +#include "cache/GpuCacheMgr.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" + +namespace milvus { +namespace scheduler { + +void +FaissIVFSQ8Pass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } +} + +bool +FaissIVFSQ8Pass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { + return false; + } + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h similarity index 81% rename from core/src/scheduler/optimizer/OnlyCPUPass.h rename to core/src/scheduler/optimizer/FaissIVFSQ8Pass.h index 253775f77e..c5853283f7 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -32,9 +33,9 @@ namespace milvus { namespace scheduler { -class OnlyCPUPass : public Pass { +class FaissIVFSQ8Pass : public Pass { public: - OnlyCPUPass() = default; + FaissIVFSQ8Pass() = default; public: void @@ -42,9 +43,14 @@ class OnlyCPUPass : public Pass { bool Run(const TaskPtr& task) override; + + private: + int32_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; }; -using OnlyCPUPassPtr = std::shared_ptr; +using FaissIVFSQ8PassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/HybridPass.cpp b/core/src/scheduler/optimizer/HybridPass.cpp deleted file mode 100644 index 8bf0d8990d..0000000000 --- a/core/src/scheduler/optimizer/HybridPass.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "scheduler/optimizer/HybridPass.h" -#include "scheduler/SchedInst.h" -#include "scheduler/task/SearchTask.h" -#include "scheduler/tasklabel/SpecResLabel.h" - -namespace milvus { -namespace scheduler { - -void -HybridPass::Init() { -} - -bool -HybridPass::Run(const TaskPtr& task) { - // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu - if (task->Type() != TaskType::SearchTask) - return false; - auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ == (int)engine::EngineType::FAISS_IVFSQ8H) { - // TODO: remove "cpu" hardcode - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); - auto label = std::make_shared(std::weak_ptr(res_ptr)); - task->label() = label; - return true; - } - return false; -} - -} // namespace scheduler -} // namespace milvus From a176eb94a21b2cf015745167df0903f490709219 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Tue, 19 Nov 2019 21:12:12 +0800 Subject: [PATCH 34/82] clange format --- core/src/scheduler/SchedInst.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 912f5edf8d..dc2d5ade35 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -23,11 +23,11 @@ #include "Scheduler.h" #include "Utils.h" #include "optimizer/BuildIndexPass.h" -#include "optimizer/FallbackPass.h" #include "optimizer/FaissFlatPass.h" #include "optimizer/FaissIVFFlatPass.h" -#include "optimizer/FaissIVFSQ8Pass.h" #include "optimizer/FaissIVFSQ8HPass.h" +#include "optimizer/FaissIVFSQ8Pass.h" +#include "optimizer/FallbackPass.h" #include "optimizer/Optimizer.h" #include "server/Config.h" From 37df2b6fc42e1647f8dd760eacdfeb08e60210a4 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 10:22:42 +0800 Subject: [PATCH 35/82] #346 rename gpu_resource_config.enable_gpu to gpu_resource_config.enable --- core/conf/server_cpu_config.template | 2 +- core/conf/server_gpu_config.template | 2 +- core/src/server/Config.cpp | 57 ++++++++++++++-------------- core/src/server/Config.h | 10 ++--- core/unittest/server/test_config.cpp | 6 +-- 5 files changed, 38 insertions(+), 39 deletions(-) diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 1a384f19a1..41889f5cef 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -36,7 +36,7 @@ engine_config: gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only gpu_resource_config: - enable_gpu: false # whether to enable GPU resources + enable: false # whether to enable GPU resources cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer search_resources: # define the GPU devices used for search computation, must be in format gpux - gpu0 diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 9790c93491..531c633da7 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -36,7 +36,7 @@ engine_config: gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only gpu_resource_config: - enable_gpu: true # whether to enable GPU resources + enable: true # whether to enable GPU resources cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer search_resources: # define the GPU devices used for search computation, must be in format gpux - gpu0 diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index e3a48b6ae4..2ff91bfe3a 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -190,8 +190,8 @@ Config::ValidateConfig() { /* gpu resource config */ #ifdef MILVUS_GPU_VERSION - bool resource_enable_gpu; - s = GetGpuResourceConfigEnableGpu(resource_enable_gpu); + bool gpu_resource_enable; + s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } @@ -330,7 +330,7 @@ Config::ResetDefaultConfig() { /* gpu resource config */ #ifdef MILVUS_GPU_VERSION - s = SetGpuResourceConfigEnableGpu(CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT); + s = SetGpuResourceConfigEnable(CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); if (!s.ok()) { return s; } @@ -641,10 +641,10 @@ Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { } Status -Config::CheckGpuResourceConfigEnableGpu(const std::string& value) { +Config::CheckGpuResourceConfigEnable(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { - std::string msg = "Invalid gpu resource config: " + value + - ". Possible reason: gpu_resource_config.enable_gpu is not a boolean."; + std::string msg = + "Invalid gpu resource config: " + value + ". Possible reason: gpu_resource_config.enable is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -992,10 +992,9 @@ Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { } Status -Config::GetGpuResourceConfigEnableGpu(bool& value) { - std::string str = - GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE_GPU, CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT); - Status s = CheckGpuResourceConfigEnableGpu(str); +Config::GetGpuResourceConfigEnable(bool& value) { + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); + Status s = CheckGpuResourceConfigEnable(str); if (!s.ok()) { return s; } @@ -1006,13 +1005,13 @@ Config::GetGpuResourceConfigEnableGpu(bool& value) { Status Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, @@ -1027,13 +1026,13 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { Status Config::GetGpuResourceConfigCacheThreshold(float& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, @@ -1048,13 +1047,13 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) { Status Config::GetGpuResourceConfigSearchResources(std::vector& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, @@ -1073,13 +1072,13 @@ Config::GetGpuResourceConfigSearchResources(std::vector& value) { Status Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = @@ -1295,12 +1294,12 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { /* gpu resource config */ Status -Config::SetGpuResourceConfigEnableGpu(const std::string& value) { - Status s = CheckGpuResourceConfigEnableGpu(value); +Config::SetGpuResourceConfigEnable(const std::string& value) { + Status s = CheckGpuResourceConfigEnable(value); if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE_GPU, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, value); return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 0442ae0626..7c141be57e 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -85,9 +85,9 @@ static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; /* gpu resource config */ static const char* CONFIG_GPU_RESOURCE = "gpu_resource_config"; -static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU = "enable_gpu"; +static const char* CONFIG_GPU_RESOURCE_ENABLE = "enable"; #ifdef MILVUS_GPU_VERSION -static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "true"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; #else static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "false"; #endif @@ -175,7 +175,7 @@ class Config { /* gpu resource config */ Status - CheckGpuResourceConfigEnableGpu(const std::string& value); + CheckGpuResourceConfigEnable(const std::string& value); Status CheckGpuResourceConfigCacheCapacity(const std::string& value); Status @@ -244,7 +244,7 @@ class Config { /* gpu resource config */ Status - GetGpuResourceConfigEnableGpu(bool& value); + GetGpuResourceConfigEnable(bool& value); Status GetGpuResourceConfigCacheCapacity(int64_t& value); Status @@ -305,7 +305,7 @@ class Config { /* gpu resource config */ Status - SetGpuResourceConfigEnableGpu(const std::string& value); + SetGpuResourceConfigEnable(const std::string& value); Status SetGpuResourceConfigCacheCapacity(const std::string& value); Status diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 6ba7f19160..e0fde62916 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -246,9 +246,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { /* gpu resource config */ bool resource_enable_gpu = true; - s = config.SetGpuResourceConfigEnableGpu(std::to_string(resource_enable_gpu)); + s = config.SetGpuResourceConfigEnable(std::to_string(resource_enable_gpu)); ASSERT_TRUE(s.ok()); - s = config.GetGpuResourceConfigEnableGpu(bool_val); + s = config.GetGpuResourceConfigEnable(bool_val); ASSERT_TRUE(s.ok()); ASSERT_TRUE(bool_val == resource_enable_gpu); @@ -394,7 +394,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(s.ok()); /* gpu resource config */ - s = config.SetGpuResourceConfigEnableGpu("ok"); + s = config.SetGpuResourceConfigEnable("ok"); ASSERT_FALSE(s.ok()); #ifdef MILVUS_GPU_VERSION From f9ece14f0aa57b9465e5886f84f631b2762b18b2 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Fri, 15 Nov 2019 17:00:53 +0800 Subject: [PATCH 36/82] ignore proto python --- core/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/.gitignore b/core/.gitignore index 74e41dba6b..8db8df41db 100644 --- a/core/.gitignore +++ b/core/.gitignore @@ -9,3 +9,5 @@ output.info output_new.info server.info *.pyc +src/grpc/python_gen.h +src/grpc/python/ From 660953afa2d55ec7ba3d31f02a73e7d750803d95 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Sat, 16 Nov 2019 09:34:17 +0800 Subject: [PATCH 37/82] modify sdk --- .../sdk/examples/partition/src/ClientTest.cpp | 65 ++++++++++--------- core/src/sdk/examples/simple/main.cpp | 2 +- .../sdk/examples/simple/src/ClientTest.cpp | 3 +- core/src/sdk/grpc/ClientProxy.cpp | 30 ++++++--- core/src/sdk/grpc/ClientProxy.h | 2 +- 5 files changed, 58 insertions(+), 44 deletions(-) diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index 775e1f6d60..b67b1fe907 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -31,45 +31,45 @@ namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); + const char *TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; -constexpr int64_t BATCH_ROW_COUNT = 10000; -constexpr int64_t NQ = 5; -constexpr int64_t TOP_K = 10; -constexpr int64_t NPROBE = 32; -constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different -constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; -constexpr int32_t N_LIST = 15000; -constexpr int32_t PARTITION_COUNT = 5; -constexpr int32_t TARGET_PARTITION = 3; + constexpr int64_t TABLE_DIMENSION = 512; + constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; + constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; + constexpr int64_t BATCH_ROW_COUNT = 10000; + constexpr int64_t NQ = 5; + constexpr int64_t TOP_K = 10; + constexpr int64_t NPROBE = 32; + constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different + constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; + constexpr int32_t N_LIST = 15000; + constexpr int32_t PARTITION_COUNT = 5; + constexpr int32_t TARGET_PARTITION = 3; -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; -} + milvus::TableSchema + BuildTableSchema() { + milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; + return tb_schema; + } -milvus::PartitionParam -BuildPartitionParam(int32_t index) { - std::string tag = std::to_string(index); - std::string partition_name = std::string(TABLE_NAME) + "_" + tag; - milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; - return partition_param; -} + milvus::PartitionParam + BuildPartitionParam(int32_t index) { + std::string tag = std::to_string(index); + std::string partition_name = std::string(TABLE_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; + return partition_param; + } -milvus::IndexParam -BuildIndexParam() { - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; - return index_param; -} + milvus::IndexParam + BuildIndexParam() { + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + return index_param; + } } // namespace void -ClientTest::Test(const std::string& address, const std::string& port) { +ClientTest::Test(const std::string &address, const std::string &port) { std::shared_ptr conn = milvus::Connection::Create(); milvus::Status stat; @@ -78,7 +78,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - +#ifdef yhz { // create table milvus::TableSchema tb_schema = BuildTableSchema(); stat = conn->CreateTable(tb_schema); @@ -202,4 +202,5 @@ ClientTest::Test(const std::string& address, const std::string& port) { } milvus::Connection::Destroy(conn); +#endif } diff --git a/core/src/sdk/examples/simple/main.cpp b/core/src/sdk/examples/simple/main.cpp index c08741606c..d9b6194329 100644 --- a/core/src/sdk/examples/simple/main.cpp +++ b/core/src/sdk/examples/simple/main.cpp @@ -36,7 +36,7 @@ main(int argc, char* argv[]) { {nullptr, 0, nullptr, 0}}; int option_index = 0; - std::string address = "127.0.0.1", port = "19530"; + std::string address = "192.168.1.89", port = "19530"; app_name = argv[0]; int value; diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp index dfa5e2219e..da43f2f2aa 100644 --- a/core/src/sdk/examples/simple/src/ClientTest.cpp +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -68,7 +68,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - +#ifdef yhz { // server version std::string version = conn->ServerVersion(); std::cout << "Server version: " << version << std::endl; @@ -206,4 +206,5 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; } +#endif } diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 4a9c319b4d..1805ae81fe 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -43,16 +43,28 @@ Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; - channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - if (channel_ != nullptr) { - connected_ = true; - client_ptr_ = std::make_shared(channel_); - return Status::OK(); - } +// channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - std::string reason = "connect failed!"; - connected_ = false; - return Status(StatusCode::NotConnected, reason); +// channel_ = std::make_shared(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); +// channel_ = std::shared_ptr(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); + auto uri_str = uri.c_str(); + grpc_channel * channel = grpc_insecure_channel_create(uri_str, nullptr, nullptr); +// grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr); + auto state = grpc_channel_check_connectivity_state(channel, true); + if (state == GRPC_CHANNEL_READY) { + std::cout << "Connect " << uri << " successfully"; + } else { + std::cout << "Connect " << uri << " failed."; + } +// if (channel_ != nullptr) { +// connected_ = true; +// client_ptr_ = std::make_shared(channel_); +// return Status::OK(); +// } + +// std::string reason = "connect failed!"; +// connected_ = false; +// return Status(StatusCode::NotConnected, reason); } Status diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index e332266acf..572b782769 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -105,7 +105,7 @@ class ClientProxy : public Connection { DropPartition(const PartitionParam& partition_param) override; private: - std::shared_ptr<::grpc::Channel> channel_; + std::shared_ptr channel_; private: std::shared_ptr client_ptr_; From 83d9bf69662f4bbbbe51959be08f8858b1f3485d Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Tue, 19 Nov 2019 17:37:13 +0800 Subject: [PATCH 38/82] modify shards for v0.5.3 --- shards/mishards/connections.py | 10 +++ shards/mishards/service_handler.py | 107 +++++++++++++++++++++-------- 2 files changed, 89 insertions(+), 28 deletions(-) diff --git a/shards/mishards/connections.py b/shards/mishards/connections.py index 618690a099..50e214ec9a 100644 --- a/shards/mishards/connections.py +++ b/shards/mishards/connections.py @@ -2,6 +2,7 @@ import logging import threading from functools import wraps from milvus import Milvus +from milvus.client.hooks import BaseaSearchHook from mishards import (settings, exceptions) from utils import singleton @@ -9,6 +10,12 @@ from utils import singleton logger = logging.getLogger(__name__) +class Searchook(BaseaSearchHook): + + def on_response(self, *args, **kwargs): + return True + + class Connection: def __init__(self, name, uri, max_retry=1, error_handlers=None, **kwargs): self.name = name @@ -18,6 +25,9 @@ class Connection: self.conn = Milvus() self.error_handlers = [] if not error_handlers else error_handlers self.on_retry_func = kwargs.get('on_retry_func', None) + + # define search hook + self.conn._set_hook(search_in_file=Searchook()) # self._connect() def __str__(self): diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 2f19152ae6..620f6213de 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -29,39 +29,88 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): self.router = router self.max_workers = max_workers + def _reduce(self, source_ids, ids, source_diss, diss, k, reverse): + if source_diss[k - 1] <= diss[0]: + return source_ids, source_diss + if diss[k - 1] <= source_diss[0]: + return ids, diss + + diss_t = enumerate(source_diss.extend(diss)) + diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] + diss_m_out = [id_ for _, id_ in diss_m_rst] + + id_t = source_ids.extend(ids) + id_m_out = [id_t[i] for i, _ in diss_m_rst] + + return id_m_out, diss_m_out + def _do_merge(self, files_n_topk_results, topk, reverse=False, **kwargs): status = status_pb2.Status(error_code=status_pb2.SUCCESS, reason="Success") if not files_n_topk_results: return status, [] - request_results = defaultdict(list) + # request_results = defaultdict(list) + # row_num = files_n_topk_results[0].row_num + merge_id_results = [] + merge_dis_results = [] calc_time = time.time() for files_collection in files_n_topk_results: if isinstance(files_collection, tuple): status, _ = files_collection return status, [] - for request_pos, each_request_results in enumerate( - files_collection.topk_query_result): - request_results[request_pos].extend( - each_request_results.query_result_arrays) - request_results[request_pos] = sorted( - request_results[request_pos], - key=lambda x: x.distance, - reverse=reverse)[:topk] + + row_num = files_collection.row_num + ids = files_collection.ids + diss = files_collection.distances # distance collections + batch_len = len(ids) // row_num + + for row_index in range(row_num): + id_batch = ids[row_index * batch_len: (row_index + 1) * batch_len] + dis_batch = diss[row_index * batch_len: (row_index + 1) * batch_len] + + if len(merge_id_results) < row_index: + raise ValueError("merge error") + elif len(merge_id_results) == row_index: + # TODO: may bug here + merge_id_results.append(id_batch) + merge_dis_results.append(dis_batch) + else: + merge_id_results[row_index].extend(ids[row_index * batch_len, (row_index + 1) * batch_len]) + merge_dis_results[row_index].extend(diss[row_index * batch_len, (row_index + 1) * batch_len]) + # _reduce(_ids, _diss, k, reverse) + merge_id_results[row_index], merge_dis_results[row_index] = \ + self._reduce(merge_id_results[row_index], id_batch, + merge_dis_results[row_index], dis_batch, + batch_len, + reverse) + + # for request_pos, each_request_results in enumerate( + # files_collection.topk_query_result): + # request_results[request_pos].extend( + # each_request_results.query_result_arrays) + # request_results[request_pos] = sorted( + # request_results[request_pos], + # key=lambda x: x.distance, + # reverse=reverse)[:topk] calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - results = sorted(request_results.items()) - topk_query_result = [] + # results = sorted(request_results.items()) + id_mrege_list = [] + dis_mrege_list = [] - for result in results: - query_result = TopKQueryResult(query_result_arrays=result[1]) - topk_query_result.append(query_result) + for id_results, dis_results in zip(merge_id_results, merge_dis_results): + id_mrege_list.extend(id_results) + dis_mrege_list.extend(dis_results) - return status, topk_query_result + # for result in results: + # query_result = TopKQueryResult(query_result_arrays=result[1]) + # topk_query_result.append(query_result) + + return status, id_mrege_list, dis_mrege_list def _do_query(self, context, @@ -109,8 +158,8 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): file_ids=query_params['file_ids'], query_records=vectors, top_k=topk, - nprobe=nprobe, - lazy_=True) + nprobe=nprobe + ) end = time.time() logger.info('search_vectors_in_files takes: {}'.format(end - start)) @@ -241,7 +290,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): logger.info('Search {}: topk={} nprobe={}'.format( table_name, topk, nprobe)) - metadata = {'resp_class': milvus_pb2.TopKQueryResultList} + metadata = {'resp_class': milvus_pb2.TopKQueryResult} if nprobe > self.MAX_NPROBE or nprobe <= 0: raise exceptions.InvalidArgumentError( @@ -275,22 +324,24 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): query_range_array.append( Range(query_range.start_value, query_range.end_value)) - status, results = self._do_query(context, - table_name, - table_meta, - query_record_array, - topk, - nprobe, - query_range_array, - metadata=metadata) + status, id_results, dis_results = self._do_query(context, + table_name, + table_meta, + query_record_array, + topk, + nprobe, + query_range_array, + metadata=metadata) now = time.time() logger.info('SearchVector takes: {}'.format(now - start)) - topk_result_list = milvus_pb2.TopKQueryResultList( + topk_result_list = milvus_pb2.TopKQueryResult( status=status_pb2.Status(error_code=status.error_code, reason=status.reason), - topk_query_result=results) + row_num=len(query_record_array), + ids=id_results, + distances=dis_results) return topk_result_list @mark_grpc_method From 2f8be3d0586625d65f08622933d85fa8eabd6553 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Tue, 19 Nov 2019 20:36:08 +0800 Subject: [PATCH 39/82] finish results reduce in mishards --- shards/mishards/service_handler.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 620f6213de..640ae61ba8 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -34,13 +34,14 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): return source_ids, source_diss if diss[k - 1] <= source_diss[0]: return ids, diss - - diss_t = enumerate(source_diss.extend(diss)) + + source_diss.extend(diss) + diss_t = enumerate(source_diss) diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] diss_m_out = [id_ for _, id_ in diss_m_rst] - id_t = source_ids.extend(ids) - id_m_out = [id_t[i] for i, _ in diss_m_rst] + source_ids.extend(ids) + id_m_out = [source_ids[i] for i, _ in diss_m_rst] return id_m_out, diss_m_out @@ -50,8 +51,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): if not files_n_topk_results: return status, [] - # request_results = defaultdict(list) - # row_num = files_n_topk_results[0].row_num merge_id_results = [] merge_dis_results = [] @@ -64,6 +63,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): row_num = files_collection.row_num ids = files_collection.ids diss = files_collection.distances # distance collections + # TODO: batch_len is equal to topk batch_len = len(ids) // row_num for row_index in range(row_num): @@ -77,28 +77,16 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): merge_id_results.append(id_batch) merge_dis_results.append(dis_batch) else: - merge_id_results[row_index].extend(ids[row_index * batch_len, (row_index + 1) * batch_len]) - merge_dis_results[row_index].extend(diss[row_index * batch_len, (row_index + 1) * batch_len]) - # _reduce(_ids, _diss, k, reverse) merge_id_results[row_index], merge_dis_results[row_index] = \ self._reduce(merge_id_results[row_index], id_batch, merge_dis_results[row_index], dis_batch, batch_len, reverse) - # for request_pos, each_request_results in enumerate( - # files_collection.topk_query_result): - # request_results[request_pos].extend( - # each_request_results.query_result_arrays) - # request_results[request_pos] = sorted( - # request_results[request_pos], - # key=lambda x: x.distance, - # reverse=reverse)[:topk] calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - # results = sorted(request_results.items()) id_mrege_list = [] dis_mrege_list = [] @@ -106,10 +94,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): id_mrege_list.extend(id_results) dis_mrege_list.extend(dis_results) - # for result in results: - # query_result = TopKQueryResult(query_result_arrays=result[1]) - # topk_query_result.append(query_result) - return status, id_mrege_list, dis_mrege_list def _do_query(self, From b552b77a7eb6aaec2afc4ce67f452a00adea982d Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 10:53:28 +0800 Subject: [PATCH 40/82] [skip ci] add comments in shards --- shards/mishards/service_handler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 640ae61ba8..fc0ee0fa2b 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -63,7 +63,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): row_num = files_collection.row_num ids = files_collection.ids diss = files_collection.distances # distance collections - # TODO: batch_len is equal to topk + # TODO: batch_len is equal to topk, may need to compare with topk batch_len = len(ids) // row_num for row_index in range(row_num): @@ -83,7 +83,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): batch_len, reverse) - calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) From 631cf7143976f6e08bc4afb7064b46b773af8010 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 10:57:01 +0800 Subject: [PATCH 41/82] #346 fix CPU version build error --- core/src/server/Config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 7c141be57e..a6e78671b3 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -89,7 +89,7 @@ static const char* CONFIG_GPU_RESOURCE_ENABLE = "enable"; #ifdef MILVUS_GPU_VERSION static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; #else -static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "false"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "false"; #endif static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY = "cache_capacity"; static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "4"; From a5a41212f8489e3e69b4249dfcc1c1b42a13d513 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 11:06:24 +0800 Subject: [PATCH 42/82] remove test change' --- .../sdk/examples/partition/src/ClientTest.cpp | 3 +- core/src/sdk/examples/simple/main.cpp | 2 +- .../sdk/examples/simple/src/ClientTest.cpp | 3 +- core/src/sdk/grpc/ClientProxy.cpp | 28 ++++++------------- core/src/sdk/grpc/ClientProxy.h | 2 +- 5 files changed, 12 insertions(+), 26 deletions(-) diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index b67b1fe907..79ed7646b9 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -78,7 +78,7 @@ ClientTest::Test(const std::string &address, const std::string &port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } -#ifdef yhz + { // create table milvus::TableSchema tb_schema = BuildTableSchema(); stat = conn->CreateTable(tb_schema); @@ -202,5 +202,4 @@ ClientTest::Test(const std::string &address, const std::string &port) { } milvus::Connection::Destroy(conn); -#endif } diff --git a/core/src/sdk/examples/simple/main.cpp b/core/src/sdk/examples/simple/main.cpp index d9b6194329..c08741606c 100644 --- a/core/src/sdk/examples/simple/main.cpp +++ b/core/src/sdk/examples/simple/main.cpp @@ -36,7 +36,7 @@ main(int argc, char* argv[]) { {nullptr, 0, nullptr, 0}}; int option_index = 0; - std::string address = "192.168.1.89", port = "19530"; + std::string address = "127.0.0.1", port = "19530"; app_name = argv[0]; int value; diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp index da43f2f2aa..dfa5e2219e 100644 --- a/core/src/sdk/examples/simple/src/ClientTest.cpp +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -68,7 +68,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } -#ifdef yhz + { // server version std::string version = conn->ServerVersion(); std::cout << "Server version: " << version << std::endl; @@ -206,5 +206,4 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; } -#endif } diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 1805ae81fe..4a9c319b4d 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -43,28 +43,16 @@ Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; -// channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - -// channel_ = std::make_shared(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); -// channel_ = std::shared_ptr(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); - auto uri_str = uri.c_str(); - grpc_channel * channel = grpc_insecure_channel_create(uri_str, nullptr, nullptr); -// grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr); - auto state = grpc_channel_check_connectivity_state(channel, true); - if (state == GRPC_CHANNEL_READY) { - std::cout << "Connect " << uri << " successfully"; - } else { - std::cout << "Connect " << uri << " failed."; + channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); + if (channel_ != nullptr) { + connected_ = true; + client_ptr_ = std::make_shared(channel_); + return Status::OK(); } -// if (channel_ != nullptr) { -// connected_ = true; -// client_ptr_ = std::make_shared(channel_); -// return Status::OK(); -// } -// std::string reason = "connect failed!"; -// connected_ = false; -// return Status(StatusCode::NotConnected, reason); + std::string reason = "connect failed!"; + connected_ = false; + return Status(StatusCode::NotConnected, reason); } Status diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index 572b782769..e332266acf 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -105,7 +105,7 @@ class ClientProxy : public Connection { DropPartition(const PartitionParam& partition_param) override; private: - std::shared_ptr channel_; + std::shared_ptr<::grpc::Channel> channel_; private: std::shared_ptr client_ptr_; From ed5219b72804d86fc910495418411aa981348a49 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 11:23:48 +0800 Subject: [PATCH 43/82] [skip ci] foramt pass --- .../sdk/examples/partition/src/ClientTest.cpp | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index 79ed7646b9..775e1f6d60 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -31,45 +31,45 @@ namespace { - const char *TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); +const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); - constexpr int64_t TABLE_DIMENSION = 512; - constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; - constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; - constexpr int64_t BATCH_ROW_COUNT = 10000; - constexpr int64_t NQ = 5; - constexpr int64_t TOP_K = 10; - constexpr int64_t NPROBE = 32; - constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different - constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; - constexpr int32_t N_LIST = 15000; - constexpr int32_t PARTITION_COUNT = 5; - constexpr int32_t TARGET_PARTITION = 3; +constexpr int64_t TABLE_DIMENSION = 512; +constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; +constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; +constexpr int64_t BATCH_ROW_COUNT = 10000; +constexpr int64_t NQ = 5; +constexpr int64_t TOP_K = 10; +constexpr int64_t NPROBE = 32; +constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different +constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; +constexpr int32_t N_LIST = 15000; +constexpr int32_t PARTITION_COUNT = 5; +constexpr int32_t TARGET_PARTITION = 3; - milvus::TableSchema - BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; - } +milvus::TableSchema +BuildTableSchema() { + milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; + return tb_schema; +} - milvus::PartitionParam - BuildPartitionParam(int32_t index) { - std::string tag = std::to_string(index); - std::string partition_name = std::string(TABLE_NAME) + "_" + tag; - milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; - return partition_param; - } +milvus::PartitionParam +BuildPartitionParam(int32_t index) { + std::string tag = std::to_string(index); + std::string partition_name = std::string(TABLE_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; + return partition_param; +} - milvus::IndexParam - BuildIndexParam() { - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; - return index_param; - } +milvus::IndexParam +BuildIndexParam() { + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + return index_param; +} } // namespace void -ClientTest::Test(const std::string &address, const std::string &port) { +ClientTest::Test(const std::string& address, const std::string& port) { std::shared_ptr conn = milvus::Connection::Create(); milvus::Status stat; From 359197eaafad45e8b0d32df1cc1726b7ac2f8bd3 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 11:35:36 +0800 Subject: [PATCH 44/82] update requirements.txt --- shards/requirements.txt | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/shards/requirements.txt b/shards/requirements.txt index 14bdde2a06..d17c13ddc9 100644 --- a/shards/requirements.txt +++ b/shards/requirements.txt @@ -1,21 +1,41 @@ +astroid==2.3.3 +atomicwrites==1.3.0 +attrs==19.3.0 +cachetools==3.1.1 +certifi==2019.9.11 +chardet==3.0.4 environs==4.2.0 factory-boy==2.12.0 Faker==1.0.7 fire==0.1.3 google-auth==1.6.3 grpcio==1.22.0 +grpcio-opentracing==1.1.4 grpcio-tools==1.22.0 +idna==2.8 +importlib-metadata==0.23 +isort==4.3.21 +jaeger-client==4.1.0 kubernetes==10.0.1 +lazy-object-proxy==1.4.3 MarkupSafe==1.1.1 marshmallow==2.19.5 -pymysql==0.9.3 +mccabe==0.6.1 +mock==2.0.0 +more-itertools==7.2.0 +oauthlib==3.1.0 +opentracing==2.2.0 +packaging==19.2 +pbr==5.4.3 +pluggy==0.13.0 +pluginbase==1.0.0 protobuf==3.9.1 py==1.8.0 pyasn1==0.4.7 pyasn1-modules==0.2.6 pylint==2.3.1 -pymilvus-test==0.2.28 -#pymilvus==0.2.0 +pymilvus==0.2.5 +PyMySQL==0.9.3 pyparsing==2.4.0 pytest==4.6.3 pytest-level==0.1.1 @@ -25,13 +45,19 @@ pytest-timeout==1.3.3 python-dateutil==2.8.0 python-dotenv==0.10.3 pytz==2019.1 +PyYAML==5.1.2 requests==2.22.0 requests-oauthlib==1.2.0 rsa==4.0 six==1.12.0 SQLAlchemy==1.3.5 +text-unidecode==1.2 +threadloop==1.0.2 +thrift==0.13.0 +tornado==5.1.1 +typed-ast==1.4.0 urllib3==1.25.3 -jaeger-client>=3.4.0 -grpcio-opentracing>=1.0 -mock==2.0.0 -pluginbase==1.0.0 +wcwidth==0.1.7 +websocket-client==0.56.0 +wrapt==1.11.2 +zipp==0.6.0 From 692b2110315fc093ab908d44059875417ec5f4ec Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 11:51:23 +0800 Subject: [PATCH 45/82] update requirements only pymilvus --- shards/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shards/requirements.txt b/shards/requirements.txt index 14bdde2a06..4ea03e0b17 100644 --- a/shards/requirements.txt +++ b/shards/requirements.txt @@ -14,7 +14,7 @@ py==1.8.0 pyasn1==0.4.7 pyasn1-modules==0.2.6 pylint==2.3.1 -pymilvus-test==0.2.28 +pymilvus==0.2.5 #pymilvus==0.2.0 pyparsing==2.4.0 pytest==4.6.3 From 15dcce417e3c2b68785d09075f885378ef50a4bf Mon Sep 17 00:00:00 2001 From: groot Date: Wed, 20 Nov 2019 12:24:18 +0800 Subject: [PATCH 46/82] #377 Create partition success if tag name only contains spaces --- CHANGELOG.md | 12 ++++-- core/src/db/DBImpl.cpp | 11 ++++- core/src/db/meta/MySQLMetaImpl.cpp | 40 ++++++++++++++----- core/src/db/meta/SqliteMetaImpl.cpp | 39 +++++++++++++----- .../request/CreatePartitionRequest.cpp | 2 +- .../request/DropPartitionRequest.cpp | 38 +++++++++++++----- core/src/utils/ValidationUtil.cpp | 22 ++++++++-- core/src/utils/ValidationUtil.h | 3 ++ core/unittest/db/test_db.cpp | 7 ++++ core/unittest/db/test_db_mysql.cpp | 8 ++++ 10 files changed, 141 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ced0633cdc..dcd8c161d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,24 +12,28 @@ Please mark all change in change log and use the ticket from JIRA. - \#340 - Test cases run failed on 0.6.0 - \#353 - Rename config.h.in to version.h.in - \#374 - sdk_simple return empty result +- \#377 - Create partition success if tag name only contains spaces - \#397 - sdk_simple return incorrect result +- \#399 - Create partition should be failed if partition tag existed +- \#412 - Message returned is confused when partition created with null partition name +- \#416 - Drop the same partition success repeatally ## Feature - \#12 - Pure CPU version for Milvus - \#77 - Support table partition -- \#226 - Experimental shards middleware for Milvus - \#127 - Support new Index type IVFPQ +- \#226 - Experimental shards middleware for Milvus ## Improvement +- \#255 - Add ivfsq8 test report detailed version - \#275 - Rename C++ SDK IndexType -- \#284 - Change C++ SDK to shared library - \#260 - C++ SDK README - \#266 - Rpc request source code refactor -- \#314 - add Find FAISS in CMake +- \#284 - Change C++ SDK to shared library - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue +- \#314 - add Find FAISS in CMake - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md -- \#255 - Add ivfsq8 test report detailed version - \#404 - Add virtual method Init() in Pass abstract class - \#409 - Add a Fallback pass in optimizer diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 2559b3a46b..7b0103f52e 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -279,6 +279,11 @@ DBImpl::DropPartitionByTag(const std::string& table_id, const std::string& parti std::string partition_name; auto status = meta_ptr_->GetPartitionName(table_id, partition_tag, partition_name); + if (!status.ok()) { + ENGINE_LOG_ERROR << status.message(); + return status; + } + return DropPartition(partition_name); } @@ -853,8 +858,12 @@ DBImpl::GetPartitionsByTags(const std::string& table_id, const std::vectorShowPartitions(table_id, partiton_array); for (auto& tag : partition_tags) { + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); for (auto& schema : partiton_array) { - if (server::StringHelpFunctions::IsRegexMatch(schema.partition_tag_, tag)) { + if (server::StringHelpFunctions::IsRegexMatch(schema.partition_tag_, valid_tag)) { partition_name_array.insert(schema.table_id_); } } diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index bf83447806..df994af03f 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -22,6 +22,7 @@ #include "metrics/Metrics.h" #include "utils/Exception.h" #include "utils/Log.h" +#include "utils/StringHelpFunctions.h" #include #include @@ -1165,14 +1166,20 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p return Status(DB_ERROR, "Nested partition is not allow"); } - if (partition_name == "") { - // not allow duplicated partition - std::string exist_partition; - GetPartitionName(table_id, tag, exist_partition); - if (!exist_partition.empty()) { - return Status(DB_ERROR, "Duplicated partition is not allow"); - } + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + // not allow duplicated partition + std::string exist_partition; + GetPartitionName(table_id, valid_tag, exist_partition); + if (!exist_partition.empty()) { + return Status(DB_ERROR, "Duplicated partition is not allow"); + } + + if (partition_name == "") { + // generate unique partition name NextTableId(table_schema.table_id_); } else { table_schema.table_id_ = partition_name; @@ -1182,9 +1189,14 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p table_schema.flag_ = 0; table_schema.created_on_ = utils::GetMicroSecTimeStamp(); table_schema.owner_table_ = table_id; - table_schema.partition_tag_ = tag; + table_schema.partition_tag_ = valid_tag; - return CreateTable(table_schema); + status = CreateTable(table_schema); + if (status.code() == DB_ALREADY_EXIST) { + return Status(DB_ALREADY_EXIST, "Partition already exists"); + } + + return status; } Status @@ -1231,6 +1243,12 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& try { server::MetricCollector metric; mysqlpp::StoreQueryResult res; + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); @@ -1240,7 +1258,7 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& mysqlpp::Query allPartitionsQuery = connectionPtr->query(); allPartitionsQuery << "SELECT table_id FROM " << META_TABLES << " WHERE owner_table = " << mysqlpp::quote - << table_id << " AND partition_tag = " << mysqlpp::quote << tag << " AND state <> " + << table_id << " AND partition_tag = " << mysqlpp::quote << valid_tag << " AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allPartitionsQuery.str(); @@ -1252,7 +1270,7 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& const mysqlpp::Row& resRow = res[0]; resRow["table_id"].to_string(partition_name); } else { - return Status(DB_NOT_FOUND, "Partition " + tag + " of table " + table_id + " not found"); + return Status(DB_NOT_FOUND, "Partition " + valid_tag + " of table " + table_id + " not found"); } } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN GET PARTITION NAME", e.what()); diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 22e953fe9d..ffefc4ed43 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -22,6 +22,7 @@ #include "metrics/Metrics.h" #include "utils/Exception.h" #include "utils/Log.h" +#include "utils/StringHelpFunctions.h" #include #include @@ -760,14 +761,20 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& return Status(DB_ERROR, "Nested partition is not allow"); } - if (partition_name == "") { - // not allow duplicated partition - std::string exist_partition; - GetPartitionName(table_id, tag, exist_partition); - if(!exist_partition.empty()) { - return Status(DB_ERROR, "Duplicated partition is not allow"); - } + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + // not allow duplicated partition + std::string exist_partition; + GetPartitionName(table_id, valid_tag, exist_partition); + if(!exist_partition.empty()) { + return Status(DB_ERROR, "Duplicated partition is not allow"); + } + + if (partition_name == "") { + // generate unique partition name NextTableId(table_schema.table_id_); } else { table_schema.table_id_ = partition_name; @@ -777,9 +784,14 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& table_schema.flag_ = 0; table_schema.created_on_ = utils::GetMicroSecTimeStamp(); table_schema.owner_table_ = table_id; - table_schema.partition_tag_ = tag; + table_schema.partition_tag_ = valid_tag; - return CreateTable(table_schema); + status = CreateTable(table_schema); + if (status.code() == DB_ALREADY_EXIST) { + return Status(DB_ALREADY_EXIST, "Partition already exists"); + } + + return status; } Status @@ -814,13 +826,18 @@ SqliteMetaImpl::GetPartitionName(const std::string& table_id, const std::string& try { server::MetricCollector metric; + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + auto name = ConnectorPtr->select(columns(&TableSchema::table_id_), where(c(&TableSchema::owner_table_) == table_id - and c(&TableSchema::partition_tag_) == tag)); + and c(&TableSchema::partition_tag_) == valid_tag)); if (name.size() > 0) { partition_name = std::get<0>(name[0]); } else { - return Status(DB_NOT_FOUND, "Table " + table_id + "'s partition " + tag + " not found"); + return Status(DB_NOT_FOUND, "Table " + table_id + "'s partition " + valid_tag + " not found"); } } catch (std::exception &e) { return HandleException("Encounter exception when get partition name", e.what()); diff --git a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp index bc2e7dc105..3bd4a86ef6 100644 --- a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp +++ b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp @@ -51,7 +51,7 @@ CreatePartitionRequest::OnExecute() { return status; } - status = ValidationUtil::ValidateTableName(partition_param_->partition_name()); + status = ValidationUtil::ValidatePartitionName(partition_param_->partition_name()); if (!status.ok()) { return status; } diff --git a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp index 217ca63a2a..0e29b6abe8 100644 --- a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp +++ b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp @@ -22,6 +22,7 @@ #include "utils/ValidationUtil.h" #include +#include namespace milvus { namespace server { @@ -38,23 +39,40 @@ DropPartitionRequest::Create(const ::milvus::grpc::PartitionParam* partition_par Status DropPartitionRequest::OnExecute() { - if (!partition_param_->partition_name().empty()) { - auto status = ValidationUtil::ValidateTableName(partition_param_->partition_name()); - if (!status.ok()) { - return status; - } - return DBWrapper::DB()->DropPartition(partition_param_->partition_name()); - } else { - auto status = ValidationUtil::ValidateTableName(partition_param_->table_name()); + std::string table_name = partition_param_->table_name(); + std::string partition_name = partition_param_->partition_name(); + std::string partition_tag = partition_param_->tag(); + if (!partition_name.empty()) { + auto status = ValidationUtil::ValidateTableName(partition_name); if (!status.ok()) { return status; } - status = ValidationUtil::ValidatePartitionTags({partition_param_->tag()}); + // check partition existence + engine::meta::TableSchema table_info; + table_info.table_id_ = partition_name; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, + "Table " + table_name + "'s partition " + partition_name + " not found"); + } else { + return status; + } + } + + return DBWrapper::DB()->DropPartition(partition_name); + } else { + auto status = ValidationUtil::ValidateTableName(table_name); if (!status.ok()) { return status; } - return DBWrapper::DB()->DropPartitionByTag(partition_param_->table_name(), partition_param_->tag()); + + status = ValidationUtil::ValidatePartitionTags({partition_tag}); + if (!status.ok()) { + return status; + } + return DBWrapper::DB()->DropPartitionByTag(table_name, partition_tag); } } diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index ec696ff3e0..fc52eb32a9 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -18,6 +18,7 @@ #include "utils/ValidationUtil.h" #include "Log.h" #include "db/engine/ExecutionEngine.h" +#include "utils/StringHelpFunctions.h" #include #ifdef MILVUS_GPU_VERSION @@ -168,11 +169,26 @@ ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSc return Status::OK(); } +Status +ValidationUtil::ValidatePartitionName(const std::string& partition_name) { + if (partition_name.empty()) { + std::string msg = "Partition name should not be empty."; + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_TABLE_NAME, msg); + } + + return ValidateTableName(partition_name); +} + Status ValidationUtil::ValidatePartitionTags(const std::vector& partition_tags) { - for (auto& tag : partition_tags) { - if (tag.empty()) { - std::string msg = "Invalid partition tag: " + tag + ". " + "Partition tag should not be empty."; + for (const std::string& tag : partition_tags) { + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + StringHelpFunctions::TrimStringBlank(valid_tag); + if (valid_tag.empty()) { + std::string msg = "Invalid partition tag: " + valid_tag + ". " + "Partition tag should not be empty."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_NPROBE, msg); } diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 01801e295a..9157ffad56 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -55,6 +55,9 @@ class ValidationUtil { static Status ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); + static Status + ValidatePartitionName(const std::string& partition_name); + static Status ValidatePartitionTags(const std::vector& partition_tags); diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index d8614dd5d1..343e924e8e 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -461,6 +461,13 @@ TEST_F(DBTest, PARTITION_TEST) { stat = db_->CreatePartition(table_name, partition_name, partition_tag); ASSERT_TRUE(stat.ok()); + // not allow nested partition + stat = db_->CreatePartition(partition_name, "dumy", "dummy"); + ASSERT_FALSE(stat.ok()); + + // not allow duplicated partition + stat = db_->CreatePartition(table_name, partition_name, partition_tag); + ASSERT_FALSE(stat.ok()); std::vector xb; BuildVectors(INSERT_BATCH, xb); diff --git a/core/unittest/db/test_db_mysql.cpp b/core/unittest/db/test_db_mysql.cpp index f828431838..93eea2fd27 100644 --- a/core/unittest/db/test_db_mysql.cpp +++ b/core/unittest/db/test_db_mysql.cpp @@ -297,6 +297,14 @@ TEST_F(MySqlDBTest, PARTITION_TEST) { stat = db_->CreatePartition(table_name, partition_name, partition_tag); ASSERT_TRUE(stat.ok()); + // not allow nested partition + stat = db_->CreatePartition(partition_name, "dumy", "dummy"); + ASSERT_FALSE(stat.ok()); + + // not allow duplicated partition + stat = db_->CreatePartition(table_name, partition_name, partition_tag); + ASSERT_FALSE(stat.ok()); + std::vector xb; BuildVectors(INSERT_BATCH, xb); From b65685df8f30129f102cb3e9d30debfc7f4fe9d4 Mon Sep 17 00:00:00 2001 From: groot Date: Wed, 20 Nov 2019 14:15:57 +0800 Subject: [PATCH 47/82] fix typo --- core/src/db/meta/MySQLMetaImpl.cpp | 4 ++-- core/src/db/meta/SqliteMetaImpl.cpp | 4 ++-- core/unittest/server/test_rpc.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index df994af03f..4406b87f7e 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1163,7 +1163,7 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p // not allow create partition under partition if (!table_schema.owner_table_.empty()) { - return Status(DB_ERROR, "Nested partition is not allow"); + return Status(DB_ERROR, "Nested partition is not allowed"); } // trim side-blank of tag, only compare valid characters @@ -1175,7 +1175,7 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p std::string exist_partition; GetPartitionName(table_id, valid_tag, exist_partition); if (!exist_partition.empty()) { - return Status(DB_ERROR, "Duplicated partition is not allow"); + return Status(DB_ERROR, "Duplicate partition is not allowed"); } if (partition_name == "") { diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index ffefc4ed43..12128c074d 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -758,7 +758,7 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& // not allow create partition under partition if(!table_schema.owner_table_.empty()) { - return Status(DB_ERROR, "Nested partition is not allow"); + return Status(DB_ERROR, "Nested partition is not allowed"); } // trim side-blank of tag, only compare valid characters @@ -770,7 +770,7 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& std::string exist_partition; GetPartitionName(table_id, valid_tag, exist_partition); if(!exist_partition.empty()) { - return Status(DB_ERROR, "Duplicated partition is not allow"); + return Status(DB_ERROR, "Duplicate partition is not allowed"); } if (partition_name == "") { diff --git a/core/unittest/server/test_rpc.cpp b/core/unittest/server/test_rpc.cpp index 34d4b0ad3a..5753c68422 100644 --- a/core/unittest/server/test_rpc.cpp +++ b/core/unittest/server/test_rpc.cpp @@ -409,7 +409,7 @@ TEST_F(RpcHandlerTest, PARTITION_TEST) { partition_parm.set_partition_name(partition_name); handler->DropPartition(&context, &partition_parm, &response); - ASSERT_EQ(response.error_code(), ::grpc::Status::OK.error_code()); + ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code()); } TEST_F(RpcHandlerTest, CMD_TEST) { From 58e591ae7c1d24b929f7a542fead68fdae803e3c Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 20 Nov 2019 14:19:31 +0800 Subject: [PATCH 48/82] Update Jenkinsfile --- ci/jenkins/Jenkinsfile | 40 ++++++++++++++++++++-------------------- ci/scripts/build.sh | 10 ++++++++-- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 8d3953b112..bf7a70d11a 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -50,7 +50,7 @@ pipeline { } stages { - stage("Run Build") { + stage("Run GPU Version Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -60,7 +60,7 @@ pipeline { } stages { - stage('Build') { + stage('GPU Version Build') { steps { container('milvus-build-env') { script { @@ -69,7 +69,7 @@ pipeline { } } } - stage('Code Coverage') { + stage('GPU Version Code Coverage') { steps { container('milvus-build-env') { script { @@ -78,7 +78,7 @@ pipeline { } } } - stage('Upload Package') { + stage('Upload GPU Version Package') { steps { container('milvus-build-env') { script { @@ -90,7 +90,7 @@ pipeline { } } - stage("Publish docker images") { + stage("Publish GPU Version docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -100,7 +100,7 @@ pipeline { } stages { - stage('Publish') { + stage('Publish GPU Version') { steps { container('publish-images'){ script { @@ -112,7 +112,7 @@ pipeline { } } - stage("Deploy to Development") { + stage("Deploy GPU Version to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -122,7 +122,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage("Deploy GPU Version to Dev") { steps { container('milvus-test-env') { script { @@ -132,7 +132,7 @@ pipeline { } } - stage("Dev Test") { + stage("GPU Version Dev Test") { steps { container('milvus-test-env') { script { @@ -147,7 +147,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage ("Cleanup GPU Version Dev") { steps { container('milvus-test-env') { script { @@ -180,7 +180,7 @@ pipeline { } stages { - stage("Run Build") { + stage("Run CPU Version Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -190,7 +190,7 @@ pipeline { } stages { - stage('Build') { + stage('Build CPU Version') { steps { container('milvus-build-env') { script { @@ -199,7 +199,7 @@ pipeline { } } } - stage('Code Coverage') { + stage('CPU Version Code Coverage') { steps { container('milvus-build-env') { script { @@ -208,7 +208,7 @@ pipeline { } } } - stage('Upload Package') { + stage('Upload CPU Version Package') { steps { container('milvus-build-env') { script { @@ -220,7 +220,7 @@ pipeline { } } - stage("Publish docker images") { + stage("Publish CPU Version docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -230,7 +230,7 @@ pipeline { } stages { - stage('Publish') { + stage('Publish CPU Version') { steps { container('publish-images'){ script { @@ -242,7 +242,7 @@ pipeline { } } - stage("Deploy to Development") { + stage("Deploy CPU Version to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -252,7 +252,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage("Deploy CPU Version to Dev") { steps { container('milvus-test-env') { script { @@ -262,7 +262,7 @@ pipeline { } } - stage("Dev Test") { + stage("CPU Version Dev Test") { steps { container('milvus-test-env') { script { @@ -277,7 +277,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage ("Cleanup CPU Version Dev") { steps { container('milvus-test-env') { script { diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 27962ccb54..eb5205ad26 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -16,6 +16,7 @@ BUILD_TYPE="Debug" BUILD_UNITTEST="OFF" INSTALL_PREFIX="/opt/milvus" FAISS_ROOT="" +CUSTOMIZATION="OFF" # default use origin faiss BUILD_COVERAGE="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" @@ -23,7 +24,7 @@ GPU_VERSION="OFF" WITH_MKL="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "o:t:b:f:gulcjmh" arg +while getopts "o:t:b:f:gxulcjmh" arg do case $arg in o) @@ -41,6 +42,9 @@ do g) GPU_VERSION="ON"; ;; + x) + CUSTOMIZATION="ON"; + ;; u) echo "Build and run unittest cases" ; BUILD_UNITTEST="ON"; @@ -66,6 +70,7 @@ parameter: -b: core code build directory -f: faiss root path -g: gpu version +-x: milvus customization (default: OFF) -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) -c: code coverage(default: OFF) @@ -74,7 +79,7 @@ parameter: -h: help usage: -./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-u] [-l] [-c] [-j] [-m] [-h] +./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h] " exit 0 ;; @@ -96,6 +101,7 @@ CMAKE_CMD="cmake \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ +-DCUSTOMIZATION=${CUSTOMIZATION} \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ From c89bb6699ebcfdc1239635b7775bf3a827e0970b Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 11:24:54 +0800 Subject: [PATCH 49/82] #306 Use int64 for all config integer --- core/src/db/Options.h | 2 +- core/src/db/engine/ExecutionEngineImpl.cpp | 4 +- core/src/db/engine/ExecutionEngineImpl.h | 4 +- core/src/scheduler/SchedInst.cpp | 4 +- .../scheduler/optimizer/BuildIndexPass.cpp | 2 +- core/src/scheduler/optimizer/FaissFlatPass.h | 4 +- .../scheduler/optimizer/FaissIVFFlatPass.h | 4 +- .../scheduler/optimizer/FaissIVFSQ8HPass.cpp | 2 +- .../scheduler/optimizer/FaissIVFSQ8HPass.h | 4 +- .../src/scheduler/optimizer/FaissIVFSQ8Pass.h | 4 +- core/src/server/Config.cpp | 70 +++++++++---------- core/src/server/Config.h | 16 ++--- core/src/server/DBWrapper.cpp | 8 +-- core/src/utils/CommonUtil.cpp | 2 +- core/src/utils/CommonUtil.h | 2 +- core/src/wrapper/KnowhereResource.cpp | 4 +- core/unittest/server/test_config.cpp | 45 ++++++------ core/unittest/server/test_util.cpp | 2 +- 18 files changed, 91 insertions(+), 92 deletions(-) diff --git a/core/src/db/Options.h b/core/src/db/Options.h index ebecb4de5a..91459a8d5f 100644 --- a/core/src/db/Options.h +++ b/core/src/db/Options.h @@ -33,7 +33,7 @@ static const char* ARCHIVE_CONF_DISK = "disk"; static const char* ARCHIVE_CONF_DAYS = "days"; struct ArchiveConf { - using CriteriaT = std::map; + using CriteriaT = std::map; explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string()); diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 4f2f542254..973d8b5912 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -146,7 +146,7 @@ ExecutionEngineImpl::HybridLoad() const { const std::string key = location_ + ".quantizer"; server::Config& config = server::Config::GetInstance(); - std::vector gpus; + std::vector gpus; Status s = config.GetGpuResourceConfigSearchResources(gpus); if (!s.ok()) { ENGINE_LOG_ERROR << s.message(); @@ -586,7 +586,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); - std::vector gpu_ids; + std::vector gpu_ids; Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids); for (auto id : gpu_ids) { if (gpu_num_ == id) { diff --git a/core/src/db/engine/ExecutionEngineImpl.h b/core/src/db/engine/ExecutionEngineImpl.h index 7eb304426e..da0e7cfb64 100644 --- a/core/src/db/engine/ExecutionEngineImpl.h +++ b/core/src/db/engine/ExecutionEngineImpl.h @@ -122,8 +122,8 @@ class ExecutionEngineImpl : public ExecutionEngine { int64_t dim_; std::string location_; - int32_t nlist_ = 0; - int32_t gpu_num_ = 0; + int64_t nlist_ = 0; + int64_t gpu_num_ = 0; }; } // namespace engine diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 867052ef5d..69d293f986 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -55,9 +55,9 @@ load_simple_config() { // get resources #ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); - std::vector gpu_ids; + std::vector gpu_ids; config.GetGpuResourceConfigSearchResources(gpu_ids); - std::vector build_gpu_ids; + std::vector build_gpu_ids; config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids); auto pcie = Connection("pcie", 12000); diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index cd1b7550d8..d535b9675f 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -26,7 +26,7 @@ namespace scheduler { void BuildIndexPass::Init() { server::Config& config = server::Config::GetInstance(); - std::vector build_resources; + std::vector build_resources; Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); if (!s.ok()) { throw; diff --git a/core/src/scheduler/optimizer/FaissFlatPass.h b/core/src/scheduler/optimizer/FaissFlatPass.h index 60adf62a07..f219bebdf3 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.h +++ b/core/src/scheduler/optimizer/FaissFlatPass.h @@ -45,9 +45,9 @@ class FaissFlatPass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissFlatPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.h b/core/src/scheduler/optimizer/FaissIVFFlatPass.h index 9264db32a8..2d15539014 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.h +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.h @@ -45,9 +45,9 @@ class FaissIVFFlatPass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissIVFFlatPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index b63a0f7490..a99e861e03 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -32,7 +32,7 @@ FaissIVFSQ8HPass::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { - threshold_ = std::numeric_limits::max(); + threshold_ = std::numeric_limits::max(); } s = config.GetGpuResourceConfigSearchResources(gpus); } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h index fa830ff08a..0d2892809f 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h @@ -45,9 +45,9 @@ class FaissIVFSQ8HPass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissIVFSQ8HPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h index c5853283f7..e92ea2fe4e 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h @@ -45,9 +45,9 @@ class FaissIVFSQ8Pass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissIVFSQ8PassPtr = std::shared_ptr; diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 2ff91bfe3a..f3efcff0cc 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -113,19 +113,19 @@ Config::ValidateConfig() { return s; } - int32_t db_archive_disk_threshold; + int64_t db_archive_disk_threshold; s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); if (!s.ok()) { return s; } - int32_t db_archive_days_threshold; + int64_t db_archive_days_threshold; s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); if (!s.ok()) { return s; } - int32_t db_insert_buffer_size; + int64_t db_insert_buffer_size; s = GetDBConfigInsertBufferSize(db_insert_buffer_size); if (!s.ok()) { return s; @@ -170,19 +170,19 @@ Config::ValidateConfig() { } /* engine config */ - int32_t engine_use_blas_threshold; + int64_t engine_use_blas_threshold; s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); if (!s.ok()) { return s; } - int32_t engine_omp_thread_num; + int64_t engine_omp_thread_num; s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); if (!s.ok()) { return s; } - int32_t engine_gpu_search_threshold; + int64_t engine_gpu_search_threshold; s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold); if (!s.ok()) { return s; @@ -208,13 +208,13 @@ Config::ValidateConfig() { return s; } - std::vector search_resources; + std::vector search_resources; s = GetGpuResourceConfigSearchResources(search_resources); if (!s.ok()) { return s; } - std::vector index_build_resources; + std::vector index_build_resources; s = GetGpuResourceConfigBuildIndexResources(index_build_resources); if (!s.ok()) { return s; @@ -485,7 +485,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { ". Possible reason: db_config.insert_buffer_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t buffer_size = std::stoi(value) * GB; + int64_t buffer_size = std::stoll(value) * GB; if (buffer_size <= 0) { std::string msg = "Invalid insert buffer size: " + value + ". Possible reason: db_config.insert_buffer_size is not a positive integer."; @@ -540,7 +540,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t cpu_cache_capacity = std::stoi(value) * GB; + int64_t cpu_cache_capacity = std::stoll(value) * GB; if (cpu_cache_capacity <= 0) { std::string msg = "Invalid cpu cache capacity: " + value + ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; @@ -557,7 +557,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; } - int32_t buffer_value; + int64_t buffer_value; Status s = GetDBConfigInsertBufferSize(buffer_value); if (!s.ok()) { return s; @@ -619,10 +619,10 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } - int32_t omp_thread = std::stoi(value); - uint32_t sys_thread_cnt = 8; + int64_t omp_thread = std::stoll(value); + int64_t sys_thread_cnt = 8; CommonUtil::GetSystemAvailableThreads(sys_thread_cnt); - if (omp_thread > static_cast(sys_thread_cnt)) { + if (omp_thread > sys_thread_cnt) { std::string msg = "Invalid omp thread num: " + value + ". Possible reason: engine_config.omp_thread_num exceeds system cpu cores."; return Status(SERVER_INVALID_ARGUMENT, msg); @@ -657,14 +657,14 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { ". Possible reason: gpu_resource_config.cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - uint64_t gpu_cache_capacity = std::stoi(value) * GB; - std::vector gpu_ids; + int64_t gpu_cache_capacity = std::stoll(value) * GB; + std::vector gpu_ids; Status s = GetGpuResourceConfigBuildIndexResources(gpu_ids); if (!s.ok()) { return s; } - for (int32_t gpu_id : gpu_ids) { + for (int64_t gpu_id : gpu_ids) { size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) { std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id); @@ -855,37 +855,37 @@ Config::GetDBConfigBackendUrl(std::string& value) { } Status -Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { +Config::GetDBConfigArchiveDiskThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { +Config::GetDBConfigArchiveDaysThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigInsertBufferSize(int32_t& value) { +Config::GetDBConfigInsertBufferSize(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -927,7 +927,7 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -957,37 +957,37 @@ Config::GetCacheConfigCacheInsertData(bool& value) { } Status -Config::GetEngineConfigUseBlasThreshold(int32_t& value) { +Config::GetEngineConfigUseBlasThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); Status s = CheckEngineConfigUseBlasThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigOmpThreadNum(int32_t& value) { +Config::GetEngineConfigOmpThreadNum(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { +Config::GetEngineConfigGpuSearchThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); Status s = CheckEngineConfigGpuSearchThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -1020,7 +1020,7 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -1046,7 +1046,7 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) { } Status -Config::GetGpuResourceConfigSearchResources(std::vector& value) { +Config::GetGpuResourceConfigSearchResources(std::vector& value) { bool gpu_resource_enable = false; Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { @@ -1065,13 +1065,13 @@ Config::GetGpuResourceConfigSearchResources(std::vector& value) { return s; } for (std::string& res : res_vec) { - value.push_back(std::stoi(res.substr(3))); + value.push_back(std::stoll(res.substr(3))); } return Status::OK(); } Status -Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { +Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { bool gpu_resource_enable = false; Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { @@ -1091,7 +1091,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { return s; } for (std::string& res : res_vec) { - value.push_back(std::stoi(res.substr(3))); + value.push_back(std::stoll(res.substr(3))); } return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index a6e78671b3..0907080a6f 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -210,11 +210,11 @@ class Config { Status GetDBConfigBackendUrl(std::string& value); Status - GetDBConfigArchiveDiskThreshold(int32_t& value); + GetDBConfigArchiveDiskThreshold(int64_t& value); Status - GetDBConfigArchiveDaysThreshold(int32_t& value); + GetDBConfigArchiveDaysThreshold(int64_t& value); Status - GetDBConfigInsertBufferSize(int32_t& value); + GetDBConfigInsertBufferSize(int64_t& value); Status GetDBConfigPreloadTable(std::string& value); @@ -236,11 +236,11 @@ class Config { /* engine config */ Status - GetEngineConfigUseBlasThreshold(int32_t& value); + GetEngineConfigUseBlasThreshold(int64_t& value); Status - GetEngineConfigOmpThreadNum(int32_t& value); + GetEngineConfigOmpThreadNum(int64_t& value); Status - GetEngineConfigGpuSearchThreshold(int32_t& value); + GetEngineConfigGpuSearchThreshold(int64_t& value); /* gpu resource config */ Status @@ -250,9 +250,9 @@ class Config { Status GetGpuResourceConfigCacheThreshold(float& value); Status - GetGpuResourceConfigSearchResources(std::vector& value); + GetGpuResourceConfigSearchResources(std::vector& value); Status - GetGpuResourceConfigBuildIndexResources(std::vector& value); + GetGpuResourceConfigBuildIndexResources(std::vector& value); public: /* server config */ diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index 1ff914dc69..f7a43d671c 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -89,7 +89,7 @@ DBWrapper::StartService() { } // engine config - int32_t omp_thread; + int64_t omp_thread; s = config.GetEngineConfigOmpThreadNum(omp_thread); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -100,7 +100,7 @@ DBWrapper::StartService() { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; } else { - uint32_t sys_thread_cnt = 8; + int64_t sys_thread_cnt = 8; if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { omp_thread = static_cast(ceil(sys_thread_cnt * 0.5)); omp_set_num_threads(omp_thread); @@ -108,7 +108,7 @@ DBWrapper::StartService() { } // init faiss global variable - int32_t use_blas_threshold; + int64_t use_blas_threshold; s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -119,7 +119,7 @@ DBWrapper::StartService() { // set archive config engine::ArchiveConf::CriteriaT criterial; - int32_t disk, days; + int64_t disk, days; s = config.GetDBConfigArchiveDiskThreshold(disk); if (!s.ok()) { std::cerr << s.ToString() << std::endl; diff --git a/core/src/utils/CommonUtil.cpp b/core/src/utils/CommonUtil.cpp index fbf3112aeb..26e43619fb 100644 --- a/core/src/utils/CommonUtil.cpp +++ b/core/src/utils/CommonUtil.cpp @@ -54,7 +54,7 @@ CommonUtil::GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem) { } bool -CommonUtil::GetSystemAvailableThreads(uint32_t& thread_count) { +CommonUtil::GetSystemAvailableThreads(int64_t& thread_count) { // threadCnt = std::thread::hardware_concurrency(); thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count *= THREAD_MULTIPLY_CPU; diff --git a/core/src/utils/CommonUtil.h b/core/src/utils/CommonUtil.h index 939bdd6d31..121196986a 100644 --- a/core/src/utils/CommonUtil.h +++ b/core/src/utils/CommonUtil.h @@ -30,7 +30,7 @@ class CommonUtil { static bool GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem); static bool - GetSystemAvailableThreads(uint32_t& thread_count); + GetSystemAvailableThreads(int64_t& thread_count); static bool IsFileExist(const std::string& path); diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index bf6675cafa..5a2296b16e 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -48,7 +48,7 @@ KnowhereResource::Initialize() { // get build index gpu resource server::Config& config = server::Config::GetInstance(); - std::vector build_index_gpus; + std::vector build_index_gpus; s = config.GetGpuResourceConfigBuildIndexResources(build_index_gpus); if (!s.ok()) return s; @@ -58,7 +58,7 @@ KnowhereResource::Initialize() { } // get search gpu resource - std::vector search_gpus; + std::vector search_gpus; s = config.GetGpuResourceConfigSearchResources(search_gpus); if (!s.ok()) return s; diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index e0fde62916..664a08d631 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -104,7 +104,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { milvus::server::Config& config = milvus::server::Config::GetInstance(); milvus::Status s; std::string str_val; - int32_t int32_val; int64_t int64_val; float float_val; bool bool_val; @@ -160,26 +159,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(s.ok()); ASSERT_TRUE(str_val == db_backend_url); - int32_t db_archive_disk_threshold = 100; + int64_t db_archive_disk_threshold = 100; s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDiskThreshold(int32_val); + s = config.GetDBConfigArchiveDiskThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_archive_disk_threshold); + ASSERT_TRUE(int64_val == db_archive_disk_threshold); - int32_t db_archive_days_threshold = 365; + int64_t db_archive_days_threshold = 365; s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDaysThreshold(int32_val); + s = config.GetDBConfigArchiveDaysThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_archive_days_threshold); + ASSERT_TRUE(int64_val == db_archive_days_threshold); - int32_t db_insert_buffer_size = 2; + int64_t db_insert_buffer_size = 2; s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigInsertBufferSize(int32_val); + s = config.GetDBConfigInsertBufferSize(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_insert_buffer_size); + ASSERT_TRUE(int64_val == db_insert_buffer_size); /* metric config */ bool metric_enable_monitor = false; @@ -223,26 +222,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(bool_val == cache_insert_data); /* engine config */ - int32_t engine_use_blas_threshold = 50; + int64_t engine_use_blas_threshold = 50; s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigUseBlasThreshold(int32_val); + s = config.GetEngineConfigUseBlasThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_use_blas_threshold); + ASSERT_TRUE(int64_val == engine_use_blas_threshold); - int32_t engine_omp_thread_num = 8; + int64_t engine_omp_thread_num = 8; s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigOmpThreadNum(int32_val); + s = config.GetEngineConfigOmpThreadNum(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_omp_thread_num); + ASSERT_TRUE(int64_val == engine_omp_thread_num); - int32_t engine_gpu_search_threshold = 800; + int64_t engine_gpu_search_threshold = 800; s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigGpuSearchThreshold(int32_val); + s = config.GetEngineConfigGpuSearchThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_gpu_search_threshold); + ASSERT_TRUE(int64_val == engine_gpu_search_threshold); /* gpu resource config */ bool resource_enable_gpu = true; @@ -267,7 +266,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(float_val == gpu_cache_threshold); std::vector search_resources = {"gpu0"}; - std::vector search_res_vec; + std::vector search_res_vec; std::string search_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( search_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, search_res_str); @@ -276,11 +275,11 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.GetGpuResourceConfigSearchResources(search_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < search_resources.size(); i++) { - ASSERT_TRUE(std::stoi(search_resources[i].substr(3)) == search_res_vec[i]); + ASSERT_TRUE(std::stoll(search_resources[i].substr(3)) == search_res_vec[i]); } std::vector build_index_resources = {"gpu0"}; - std::vector build_index_res_vec; + std::vector build_index_res_vec; std::string build_index_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( build_index_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, build_index_res_str); @@ -289,7 +288,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.GetGpuResourceConfigBuildIndexResources(build_index_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < build_index_resources.size(); i++) { - ASSERT_TRUE(std::stoi(build_index_resources[i].substr(3)) == build_index_res_vec[i]); + ASSERT_TRUE(std::stoll(build_index_resources[i].substr(3)) == build_index_res_vec[i]); } #endif } diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index 68400f2454..e5884cac65 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -60,7 +60,7 @@ TEST(UtilTest, COMMON_TEST) { ASSERT_GT(total_mem, 0); ASSERT_GT(free_mem, 0); - uint32_t thread_cnt = 0; + int64_t thread_cnt = 0; milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt); ASSERT_GT(thread_cnt, 0); From 82e6b2738b9b72af25b1495ff50a697cff2c5b1b Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 14:35:10 +0800 Subject: [PATCH 50/82] #306 update changelog --- CHANGELOG.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42d5b8978f..7d88bc270e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,15 +22,16 @@ Please mark all change in change log and use the ticket from JIRA. - \#346 - Support build index with multiple gpu ## Improvement -- \#275 - Rename C++ SDK IndexType -- \#284 - Change C++ SDK to shared library +- \#255 - Add ivfsq8 test report detailed version - \#260 - C++ SDK README - \#266 - Rpc request source code refactor -- \#314 - add Find FAISS in CMake +- \#275 - Rename C++ SDK IndexType +- \#284 - Change C++ SDK to shared library +- \#306 - Use int64 for all config integer - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue +- \#314 - add Find FAISS in CMake - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md -- \#255 - Add ivfsq8 test report detailed version - \#404 - Add virtual method Init() in Pass abstract class - \#409 - Add a Fallback pass in optimizer From 01cd2b1e022ffd861c0add9d9ad9c2dfdd38d13d Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 15:01:43 +0800 Subject: [PATCH 51/82] [skip ci] Add README for docs file --- docs/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 docs/README.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..64409d8240 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,11 @@ +# Docs + +This repository contains test reports on the search performance of different index types on standalone Milvus. + +The tests are run on [SIFT1B dataset](http://corpus-texmex.irisa.fr/), and provide results on the following measures: + +- Query Elapsed Time: Time cost (in seconds) to run a query. + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved. + +Test variables are `nq` and `topk`. From 94d11b99310599a0134c851194460e183013c8d6 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Wed, 20 Nov 2019 15:03:44 +0800 Subject: [PATCH 52/82] [skip ci] add ivfsq8h test report detailed version #255 --- ...us_ivfsq8h_test_report_detailed_version.md | 179 +++++++++++++++++ ...ivfsq8h_test_report_detailed_version_cn.md | 180 ++++++++++++++++++ 2 files changed, 359 insertions(+) create mode 100755 docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md create mode 100644 docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md new file mode 100755 index 0000000000..042d059684 --- /dev/null +++ b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md @@ -0,0 +1,179 @@ +# milvus_ivfsq8h_test_report_detailed_version + +## Summary + +This document contains the test reports of IVF_SQ8H index on Milvus single server. + + + +## Test objectives + +The time cost and recall when searching with different parameters. + + + +## Test method + +### Hardware/Software requirements + +Operating System: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +Memory: 503GB + +Docker version: 18.09 + +NVIDIA Driver version: 430.34 + +Milvus version: 0.5.3 + +SDK interface: Python 3.6.8 + +pymilvus version: 0.2.5 + + + +### Data model + +The data used in the tests are: + +- Data source: sift1b +- Data type: hdf5 + +For details on this dataset, please check : http://corpus-texmex.irisa.fr/ . + + + +### Measures + +- Query Elapsed Time: Time cost (in seconds) to run a query. Variables that affect Query Elapsed Time: + + - nq (Number of queried vectors) + +> Note: In the query test of query elapsed time, we will test the following parameters with different values: + > + > nq - grouped by: [1, 5, 10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]. + > + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved . Variables that affect Recall: + + - nq (Number of queried vectors) + - topk (Top k result of a query) + + > Note: In the query test of recall, we will test the following parameters with different values: + > + > nq - grouped by: [10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800], + > + > topk - grouped by: [1, 10, 100] + + + +## Test reports + +### Test environment + +Data base: sift1b-1,000,000,000 vectors, 128-dimension + +Table Attributes + +- nlist: 16384 +- metric_type: L2 + +Query configuration + +- nprobe: 32 + +Milvus configuration + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 +- gpu_search_threshold: 1200 +- search_resources: cpu, gpu0, gpu1 + +The definitions of Milvus configuration are on https://milvus.io/docs/en/reference/milvus_config/. + +Test method + +Test the query elapsed time and recall with several parameters, and once only change one parameter. + +- Whether to restart Milvus after each query: No + + + +### Performance test + +#### Data query + +**Test result** + +Query Elapsed Time + +topk = 100 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 0.34 | +| nq=5 | 0.72 | +| nq=10 | 0.91 | +| nq=50 | 1.51 | +| nq=100 | 2.49 | +| nq=200 | 4.09 | +| nq=400 | 7.32 | +| nq=600 | 10.63 | +| nq=800 | 13.84 | +| nq=1000 | 16.83 | +| nq=1200 | 18.20 | +| nq=1400 | 20.1 | +| nq=1600 | 20.0 | +| nq=1800 | 19.86 | + +When nq is 1800, the query time cost of a 128-dimension vector is around 11ms. + + + +**Conclusion** + +When nq < 1200, the query elapsed time increases quickly with nq; when nq > 1200, the query elapsed time increases much slower. It is because gpu_search_threshold is set to 1200, when nq < 1200, CPU is chosen to do the query, otherwise GPU is chosen. Compared with CPU, GPU has much more cores and stronger computing capability. When nq is large, it can better reflect GPU's advantages on computing. + +The query elapsed time consists of two parts: (1) index CPU-to-GPU copy time; (2) nprobe buckets search time. When nq is larger enough, index CPU-to-GPU copy time can be amortized efficiently. So Milvus performs well through setting suitable gpu_search_threshold. + + + +### Recall test + +**Test result** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +We use the ground_truth in sift1b dataset to calculate the recall of query results. + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=50 | 0.980 | 0.950 | 0.941 | +| nq=100 | 0.970 | 0.937 | 0.931 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | +| nq=1200 | 0.937 | 0.943 | 0.931 | +| nq=1400 | 0.939 | 0.945 | 0.931 | +| nq=1600 | 0.936 | 0.945 | 0.931 | +| nq=1800 | 0.937 | 0.946 | 0.932 | + + + +**Conclusion** + +As nq increases, the recall gradually stabilizes to over 93%. The usage of CPU or GPU and different topk are not related to recall. \ No newline at end of file diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md new file mode 100644 index 0000000000..b50d00f9bd --- /dev/null +++ b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md @@ -0,0 +1,180 @@ +# milvus_ivfsq8h_test_report_detailed_version_cn + +## 概述 + +本文描述了ivfsq8h索引在milvus单机部署方式下的测试结果。 + + + +## 测试目标 + +参数不同情况下的查询时间和召回率。 + + + +## 测试方法 + +### 软硬件环境 + +操作系统：CentOS Linux release 7.6.1810 (Core) + +CPU：Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0：GeForce GTX 1080 + +GPU1：GeForce GTX 1080 + +内存：503GB + +Docker版本：18.09 + +NVIDIA Driver版本：430.34 + +Milvus版本：0.5.3 + +SDK接口：Python 3.6.8 + +pymilvus版本：0.2.5 + + + +### 数据模型 + +本测试中用到的主要数据: + +- 数据来源：sift1b +- 数据类型：hdf5 + +关于该数据集的详细信息请参考：http://corpus-texmex.irisa.fr/ 。 + + + +### 测试指标 + +- Query Elapsed Time: 数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量： + + - nq (被查询向量的数量) + +> 备注：在向量查询测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [1, 5, 10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]的数量分组。 + > + +- Recall: 实际返回的正确结果占总数之比。影响Recall的变量： + + - nq (被查询向量的数量) + - topk (单条查询中最相似的K个结果) + + > 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]的数量分组， + > + > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。 + + + +## 测试报告 + +### 测试环境 + +数据集: sift1b-1,000,000,000向量, 128维 + +表格属性： + +- nlist: 16384 +- metric_type: L2 + +查询设置： + +- nprobe: 32 + +Milvus设置： + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 +- gpu_search_threshold: 1200 +- search_resources: cpu, gpu0, gpu1 + +Milvus设置的详细定义可以参考 https://milvus.io/docs/en/reference/milvus_config/ 。 + +测试方法 + +通过一次仅改变一个参数的值，测试查询向量时间和召回率。 + +- 查询后是否重启Milvus：否 + + + +### 性能测试 + +#### 数据查询 + +测试结果 + +Query Elapsed Time + +topk = 100 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 0.34 | +| nq=5 | 0.72 | +| nq=10 | 0.91 | +| nq=50 | 1.51 | +| nq=100 | 2.49 | +| nq=200 | 4.09 | +| nq=400 | 7.32 | +| nq=600 | 10.63 | +| nq=800 | 13.84 | +| nq=1000 | 16.83 | +| nq=1200 | 18.20 | +| nq=1400 | 20.1 | +| nq=1600 | 20.0 | +| nq=1800 | 19.86 | + +当nq为1800时，查询一条128维向量需要耗时约11毫秒。 + + + +**总结** + +当nq小于1200时，查询耗时随nq的增长快速增大；当nq大于1200时，查询耗时的增大则缓慢许多。这是因为gpu_search_threshold这一参数的值被设为1200，当nq<1200时，选择CPU进行操作，否则选择GPU进行操作。与CPU。 + +在GPU模式下的查询耗时由两部分组成：（1）索引从CPU到GPU的拷贝时间；（2）所有分桶的查询时间。当nq小于500时，索引从CPU到GPU 的拷贝时间无法被有效均摊，此时CPU模式时一个更优的选择；当nq大于500时，选择GPU模式更合理。和CPU相比，GPU具有更多的核数和更强的算力。当nq较大时，GPU在计算上的优势能被更好地被体现。 + + + +### 召回率测试 + +**测试结果** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +我们利用sift1b数据集中的ground_truth来计算查询结果的召回率。 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=50 | 0.980 | 0.950 | 0.941 | +| nq=100 | 0.970 | 0.937 | 0.931 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | +| nq=1200 | 0.937 | 0.943 | 0.931 | +| nq=1400 | 0.939 | 0.945 | 0.931 | +| nq=1600 | 0.936 | 0.945 | 0.931 | +| nq=1800 | 0.937 | 0.946 | 0.932 | + + + +**总结** + +随着nq的增大，召回率逐渐稳定至93%以上。CPU/GPU的使用以及topk的值与召回率的大小无关。 + From 8a37609dcc04440b466a616855d78d670505b770 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 15:44:25 +0800 Subject: [PATCH 53/82] [skip ci] Update README --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 08ac3de4a0..5b2fc4454b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![LICENSE](https://img.shields.io/badge/license-Apache--2.0-brightgreen) ![Language](https://img.shields.io/badge/language-C%2B%2B-blue) [![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master) -![Release](https://img.shields.io/badge/release-v0.5.2-yellowgreen) +![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen) ![Release_date](https://img.shields.io/badge/release%20date-November-yellowgreen) [中文版](README_CN.md) | [日本語版](README_JP.md) @@ -18,7 +18,7 @@ For more detailed introduction of Milvus and its architecture, see [Milvus overv Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) APIs. -Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://www.milvus.io/docs/en/release/v0.5.2/). +Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://www.milvus.io/docs/en/release/v0.5.3/). ## Get started @@ -52,12 +52,13 @@ We use [GitHub issues](https://github.com/milvus-io/milvus/issues) to track issu To connect with other users and contributors, welcome to join our [Slack channel](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk). -## Thanks +## Contributors -We greatly appreciate the help of the following people. +Below is a list of Milvus contributors. We greatly appreciate your contributions! - [akihoni](https://github.com/akihoni) provided the CN version of README, and found a broken link in the doc. - [goodhamgupta](https://github.com/goodhamgupta) fixed a filename typo in the bootcamp doc. +- [erdustiggen](https://github.com/erdustiggen) changed from std::cout to LOG for error messages, and fixed a clang format issue as well as some grammatical errors. ## Resources @@ -65,6 +66,8 @@ We greatly appreciate the help of the following people. - [Milvus bootcamp](https://github.com/milvus-io/bootcamp) +- [Milvus test reports](https://github.com/milvus-io/milvus/tree/master/docs/test_report) + - [Milvus Medium](https://medium.com/@milvusio) - [Milvus CSDN](https://zilliz.blog.csdn.net/) From e5ee302ef8058e5f5ee11e448bfa81ce3beeb891 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 16:03:43 +0800 Subject: [PATCH 54/82] [skip ci] Update README_CN --- README_CN.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README_CN.md b/README_CN.md index d5de0b1cd6..979c476ebf 100644 --- a/README_CN.md +++ b/README_CN.md @@ -4,7 +4,7 @@ ![LICENSE](https://img.shields.io/badge/license-Apache--2.0-brightgreen) ![Language](https://img.shields.io/badge/language-C%2B%2B-blue) [![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master) -![Release](https://img.shields.io/badge/release-v0.5.2-orange) +![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen) ![Release_date](https://img.shields.io/badge/release_date-October-yellowgreen) # 欢迎来到 Milvus @@ -17,7 +17,7 @@ Milvus 是一款开源的、针对海量特征向量的相似性搜索引擎。 Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java) 以及 C++ 的 API 接口。 -通过 [版本发布说明](https://milvus.io/docs/zh-CN/release/v0.5.2/) 获取最新版本的功能和更新。 +通过 [版本发布说明](https://milvus.io/docs/zh-CN/release/v0.5.3/) 获取最新版本的功能和更新。 ## 开始使用 Milvus @@ -57,6 +57,7 @@ Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java]( - [akihoni](https://github.com/akihoni) 提供了中文版 README，并发现了 README 中的无效链接。 - [goodhamgupta](https://github.com/goodhamgupta) 发现并修正了在线训练营文档中的文件名拼写错误。 +- [erdustiggen](https://github.com/erdustiggen) 将错误信息里的 std::cout 修改为 LOG，修正了一个 Clang 格式问题和一些语法错误。 ## 相关链接 @@ -64,6 +65,8 @@ Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java]( - [Milvus 在线训练营](https://github.com/milvus-io/bootcamp) +- [Milvus 测试报告](https://github.com/milvus-io/milvus/tree/master/docs/test_report_cn) + - [Milvus Medium](https://medium.com/@milvusio) - [Milvus CSDN](https://zilliz.blog.csdn.net/) From a77bd78924fb6fe0d6be82825ba79dbfaa7f969e Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Wed, 20 Nov 2019 16:05:14 +0800 Subject: [PATCH 55/82] Fix compiling error --- core/src/wrapper/VecImpl.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index 2085daed24..9918a9a025 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -21,11 +21,13 @@ #include "knowhere/index/vector_index/IndexIDMAP.h" #include "utils/Log.h" #include "wrapper/WrapperException.h" +#include "wrapper/gpu/GPUVecImpl.h" #ifdef MILVUS_GPU_VERSION -#include -#include +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" +#include "knowhere/index/vector_index/IndexIVFSQHybrid.h" #endif From bfd4c6c5224f35682725874b5c53a734fde276c4 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Wed, 20 Nov 2019 16:08:18 +0800 Subject: [PATCH 56/82] upgrade SPTAG and support KDT and BKT --- core/src/db/engine/ExecutionEngine.h | 4 +- core/src/db/engine/ExecutionEngineImpl.cpp | 8 + core/src/index/knowhere/CMakeLists.txt | 4 +- .../knowhere/index/vector_index/IndexKDT.cpp | 180 --------- .../knowhere/index/vector_index/IndexKDT.h | 87 ----- .../index/vector_index/IndexSPTAG.cpp | 348 ++++++++++++++++++ .../knowhere/index/vector_index/IndexSPTAG.h | 93 +++++ .../vector_index/helpers/IndexParameter.h | 79 +++- .../vector_index/helpers/KDTParameterMgr.cpp | 55 --- .../helpers/SPTAGParameterMgr.cpp | 75 ++++ ...{KDTParameterMgr.h => SPTAGParameterMgr.h} | 47 ++- .../AnnService/inc/Core/Common/Dataset.h | 2 +- core/src/index/unittest/CMakeLists.txt | 16 +- core/src/index/unittest/test_kdt.cpp | 144 -------- core/src/index/unittest/test_sptag.cpp | 154 ++++++++ core/src/index/unittest/utils.cpp | 14 +- core/src/wrapper/ConfAdapter.cpp | 30 ++ core/src/wrapper/ConfAdapter.h | 18 + core/src/wrapper/ConfAdapterMgr.cpp | 3 + core/src/wrapper/VecIndex.cpp | 8 +- core/src/wrapper/VecIndex.h | 4 + core/unittest/wrapper/test_wrapper.cpp | 33 +- 22 files changed, 883 insertions(+), 523 deletions(-) delete mode 100644 core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.cpp delete mode 100644 core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.h create mode 100644 core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp create mode 100644 core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h delete mode 100644 core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp create mode 100644 core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp rename core/src/index/knowhere/knowhere/index/vector_index/helpers/{KDTParameterMgr.h => SPTAGParameterMgr.h} (55%) delete mode 100644 core/src/index/unittest/test_kdt.cpp create mode 100644 core/src/index/unittest/test_sptag.cpp diff --git a/core/src/db/engine/ExecutionEngine.h b/core/src/db/engine/ExecutionEngine.h index 86a014cf66..c8784e8a90 100644 --- a/core/src/db/engine/ExecutionEngine.h +++ b/core/src/db/engine/ExecutionEngine.h @@ -35,7 +35,9 @@ enum class EngineType { NSG_MIX, FAISS_IVFSQ8H, FAISS_PQ, - MAX_VALUE = FAISS_PQ, + SPTAG_KDT, + SPTAG_BKT, + MAX_VALUE = SPTAG_BKT, }; enum class MetricType { diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 3a2b940cec..e2db7179be 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -124,6 +124,14 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { #endif break; } + case EngineType::SPTAG_KDT: { + index = GetVecIndexFactory(IndexType::SPTAG_KDT_RNT_CPU); + break; + } + case EngineType::SPTAG_BKT: { + index = GetVecIndexFactory(IndexType::SPTAG_BKT_RNT_CPU); + break; + } default: { ENGINE_LOG_ERROR << "Unsupported index type"; return nullptr; diff --git a/core/src/index/knowhere/CMakeLists.txt b/core/src/index/knowhere/CMakeLists.txt index 5f8e4d6970..285461bdef 100644 --- a/core/src/index/knowhere/CMakeLists.txt +++ b/core/src/index/knowhere/CMakeLists.txt @@ -30,10 +30,10 @@ set(external_srcs set(index_srcs knowhere/index/preprocessor/Normalize.cpp - knowhere/index/vector_index/IndexKDT.cpp + knowhere/index/vector_index/IndexSPTAG.cpp knowhere/index/vector_index/IndexIDMAP.cpp knowhere/index/vector_index/IndexIVF.cpp - knowhere/index/vector_index/helpers/KDTParameterMgr.cpp + knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp knowhere/index/vector_index/IndexNSG.cpp knowhere/index/vector_index/nsg/NSG.cpp knowhere/index/vector_index/nsg/NSGIO.cpp diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.cpp deleted file mode 100644 index 1bd45075e3..0000000000 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.cpp +++ /dev/null @@ -1,180 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#undef mkdir - -#include "knowhere/index/vector_index/IndexKDT.h" -#include "knowhere/index/vector_index/helpers/Definitions.h" -//#include "knowhere/index/preprocessor/normalize.h" -#include "knowhere/adapter/SptagAdapter.h" -#include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h" - -namespace knowhere { - -BinarySet -CPUKDTRNG::Serialize() { - std::vector index_blobs; - std::vector index_len; - - // TODO(zirui): dev - // index_ptr_->SaveIndexToMemory(index_blobs, index_len); - BinarySet binary_set; - - // - // auto sample = std::make_shared(); - // sample.reset(static_cast(index_blobs[0])); - // auto tree = std::make_shared(); - // tree.reset(static_cast(index_blobs[1])); - // auto graph = std::make_shared(); - // graph.reset(static_cast(index_blobs[2])); - // auto metadata = std::make_shared(); - // metadata.reset(static_cast(index_blobs[3])); - // - // binary_set.Append("samples", sample, index_len[0]); - // binary_set.Append("tree", tree, index_len[1]); - // binary_set.Append("graph", graph, index_len[2]); - // binary_set.Append("metadata", metadata, index_len[3]); - return binary_set; -} - -void -CPUKDTRNG::Load(const BinarySet& binary_set) { - // TODO(zirui): dev - - // std::vector index_blobs; - // - // auto samples = binary_set.GetByName("samples"); - // index_blobs.push_back(samples->data.get()); - // - // auto tree = binary_set.GetByName("tree"); - // index_blobs.push_back(tree->data.get()); - // - // auto graph = binary_set.GetByName("graph"); - // index_blobs.push_back(graph->data.get()); - // - // auto metadata = binary_set.GetByName("metadata"); - // index_blobs.push_back(metadata->data.get()); - // - // index_ptr_->LoadIndexFromMemory(index_blobs); -} - -// PreprocessorPtr -// CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { -// return std::make_shared(); -//} - -IndexModelPtr -CPUKDTRNG::Train(const DatasetPtr& origin, const Config& train_config) { - SetParameters(train_config); - DatasetPtr dataset = origin->Clone(); - - // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine - // && preprocessor_) { - // preprocessor_->Preprocess(dataset); - //} - - auto vectorset = ConvertToVectorSet(dataset); - auto metaset = ConvertToMetadataSet(dataset); - index_ptr_->BuildIndex(vectorset, metaset); - - // TODO: return IndexModelPtr - return nullptr; -} - -void -CPUKDTRNG::Add(const DatasetPtr& origin, const Config& add_config) { - SetParameters(add_config); - DatasetPtr dataset = origin->Clone(); - - // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine - // && preprocessor_) { - // preprocessor_->Preprocess(dataset); - //} - - auto vectorset = ConvertToVectorSet(dataset); - auto metaset = ConvertToMetadataSet(dataset); - index_ptr_->AddIndex(vectorset, metaset); -} - -void -CPUKDTRNG::SetParameters(const Config& config) { - for (auto& para : KDTParameterMgr::GetInstance().GetKDTParameters()) { - // auto value = config.get_with_default(para.first, para.second); - index_ptr_->SetParameter(para.first, para.second); - } -} - -DatasetPtr -CPUKDTRNG::Search(const DatasetPtr& dataset, const Config& config) { - SetParameters(config); - auto tensor = dataset->tensor()[0]; - auto p = (float*)tensor->raw_mutable_data(); - for (auto i = 0; i < 10; ++i) { - for (auto j = 0; j < 10; ++j) { - std::cout << p[i * 10 + j] << " "; - } - std::cout << std::endl; - } - std::vector query_results = ConvertToQueryResult(dataset, config); - -#pragma omp parallel for - for (auto i = 0; i < query_results.size(); ++i) { - auto target = (float*)query_results[i].GetTarget(); - std::cout << target[0] << ", " << target[1] << ", " << target[2] << std::endl; - index_ptr_->SearchIndex(query_results[i]); - } - - return ConvertToDataset(query_results); -} - -int64_t -CPUKDTRNG::Count() { - index_ptr_->GetNumSamples(); -} - -int64_t -CPUKDTRNG::Dimension() { - index_ptr_->GetFeatureDim(); -} - -VectorIndexPtr -CPUKDTRNG::Clone() { - KNOWHERE_THROW_MSG("not support"); -} - -void -CPUKDTRNG::Seal() { - // do nothing -} - -// TODO(linxj): -BinarySet -CPUKDTRNGIndexModel::Serialize() { -} - -void -CPUKDTRNGIndexModel::Load(const BinarySet& binary) { -} - -} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.h deleted file mode 100644 index f6d436995b..0000000000 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.h +++ /dev/null @@ -1,87 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include "VectorIndex.h" -#include "knowhere/index/IndexModel.h" - -namespace knowhere { - -class CPUKDTRNG : public VectorIndex { - public: - CPUKDTRNG() { - index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::KDT, SPTAG::VectorValueType::Float); - index_ptr_->SetParameter("DistCalcMethod", "L2"); - } - - public: - BinarySet - Serialize() override; - VectorIndexPtr - Clone() override; - void - Load(const BinarySet& index_array) override; - - public: - // PreprocessorPtr - // BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override; - int64_t - Count() override; - int64_t - Dimension() override; - - IndexModelPtr - Train(const DatasetPtr& dataset, const Config& config) override; - - void - Add(const DatasetPtr& dataset, const Config& config) override; - - DatasetPtr - Search(const DatasetPtr& dataset, const Config& config) override; - void - Seal() override; - - private: - void - SetParameters(const Config& config); - - private: - PreprocessorPtr preprocessor_; - std::shared_ptr index_ptr_; -}; - -using CPUKDTRNGPtr = std::shared_ptr; - -class CPUKDTRNGIndexModel : public IndexModel { - public: - BinarySet - Serialize() override; - - void - Load(const BinarySet& binary) override; - - private: - std::shared_ptr index_; -}; - -using CPUKDTRNGIndexModelPtr = std::shared_ptr; - -} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp new file mode 100644 index 0000000000..c64baebb93 --- /dev/null +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp @@ -0,0 +1,348 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +#include +#include +#include + +#undef mkdir + +#include "knowhere/index/vector_index/IndexSPTAG.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" +#include "knowhere/adapter/SptagAdapter.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/helpers/SPTAGParameterMgr.h" + +namespace knowhere { + + CPUSPTAGRNG::CPUSPTAGRNG(const std::string& IndexType) { + if (IndexType == "KDT") { + index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::KDT, SPTAG::VectorValueType::Float); + index_ptr_->SetParameter("DistCalcMethod", "L2"); + index_type_ = SPTAG::IndexAlgoType::KDT; + } else { + index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::BKT, SPTAG::VectorValueType::Float); + index_ptr_->SetParameter("DistCalcMethod", "L2"); + index_type_ = SPTAG::IndexAlgoType::BKT; + } + } + + BinarySet + CPUSPTAGRNG::Serialize() { + std::string index_config; + std::vector index_blobs; + + std::shared_ptr> buffersize = index_ptr_->CalculateBufferSize(); + std::vector res(buffersize->size() + 1); + for (uint64_t i = 1; i < res.size(); i++) { + res[i] = new char[buffersize->at(i - 1)]; + auto ptr = &res[i][0]; + index_blobs.emplace_back(SPTAG::ByteArray((std::uint8_t*)ptr, buffersize->at(i - 1), false)); + } + + index_ptr_->SaveIndex(index_config, index_blobs); + + size_t length = index_config.length(); + char* cstr = new char[length]; + snprintf(cstr, length, "%s", index_config.c_str()); + + BinarySet binary_set; + auto sample = std::make_shared(); + sample.reset(static_cast(index_blobs[0].Data())); + auto tree = std::make_shared(); + tree.reset(static_cast(index_blobs[1].Data())); + auto graph = std::make_shared(); + graph.reset(static_cast(index_blobs[2].Data())); + auto deleteid = std::make_shared(); + deleteid.reset(static_cast(index_blobs[3].Data())); + auto metadata1 = std::make_shared(); + metadata1.reset(static_cast(index_blobs[4].Data())); + auto metadata2 = std::make_shared(); + metadata2.reset(static_cast(index_blobs[5].Data())); + auto config = std::make_shared(); + config.reset(static_cast((void*)cstr)); + + binary_set.Append("samples", sample, index_blobs[0].Length()); + binary_set.Append("tree", tree, index_blobs[1].Length()); + binary_set.Append("deleteid", deleteid, index_blobs[3].Length()); + binary_set.Append("metadata1", metadata1, index_blobs[4].Length()); + binary_set.Append("metadata2", metadata2, index_blobs[5].Length()); + binary_set.Append("config", config, length); + binary_set.Append("graph", graph, index_blobs[2].Length()); + +// MemoryIOWriter writer; +// size_t len = 0; +// for (int i = 0; i < 6; ++i) { +// len = index_blobs[i].Length(); +// assert(len != 0); +// writer(&len, sizeof(size_t), 1); +// writer(index_blobs[i].Data(), len, 1); +// len = 0; +// } +// writer(&length, sizeof(size_t), 1); +// writer(cstr, length, 1); +// auto data = std::make_shared(); +// data.reset(writer.data_); +// BinarySet binary_set; +// binary_set.Append("sptag", data, writer.total); + +// MemoryIOWriter writer; +// size_t len = 0; +// for (int i = 0; i < 6; ++i) { +// if (i == 2) continue; +// len = index_blobs[i].Length(); +// assert(len != 0); +// writer(&len, sizeof(size_t), 1); +// writer(index_blobs[i].Data(), len, 1); +// len = 0; +// } +// writer(&length, sizeof(size_t), 1); +// writer(cstr, length, 1); +// auto data = std::make_shared(); +// data.reset(writer.data_); +// BinarySet binary_set; +// binary_set.Append("sptag", data, writer.total); +// auto graph = std::make_shared(); +// graph.reset(static_cast(index_blobs[2].Data())); +// binary_set.Append("graph", graph, index_blobs[2].Length()); + + return binary_set; + } + + void + CPUSPTAGRNG::Load(const BinarySet& binary_set) { + std::string index_config; + std::vector index_blobs; + + auto samples = binary_set.GetByName("samples"); + index_blobs.push_back(SPTAG::ByteArray(samples->data.get(), samples->size, false)); + + auto tree = binary_set.GetByName("tree"); + index_blobs.push_back(SPTAG::ByteArray(tree->data.get(), tree->size, false)); + + auto graph = binary_set.GetByName("graph"); + index_blobs.push_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); + + auto deleteid = binary_set.GetByName("deleteid"); + index_blobs.push_back(SPTAG::ByteArray(deleteid->data.get(), deleteid->size, false)); + + auto metadata1 = binary_set.GetByName("metadata1"); + index_blobs.push_back(SPTAG::ByteArray(metadata1->data.get(), metadata1->size, false)); + + auto metadata2 = binary_set.GetByName("metadata2"); + index_blobs.push_back(SPTAG::ByteArray(metadata2->data.get(), metadata2->size, false)); + + auto config = binary_set.GetByName("config"); + index_config = reinterpret_cast(config->data.get()); + +// std::vector index_blobs; +// auto data = binary_set.GetByName("sptag"); +// MemoryIOReader reader; +// reader.total = data->size; +// reader.data_ = data->data.get(); +// size_t len = 0; +// for (int i = 0; i < 6; ++i) { +// reader(&len, sizeof(size_t), 1); +// assert(len != 0); +// auto binary = new uint8_t[len]; +// reader(binary, len, 1); +// index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); +// len = 0; +// } +// reader(&len, sizeof(size_t), 1); +// assert(len != 0); +// auto config = new char[len]; +// reader(config, len, 1); +// std::string index_config = config; +// delete[] config; + +// std::vector index_blobs; +// auto data = binary_set.GetByName("sptag"); +// MemoryIOReader reader; +// reader.total = data->size; +// reader.data_ = data->data.get(); +// size_t len = 0; +// for (int i = 0; i < 6; ++i) { +// if (i == 2) { +// auto graph = binary_set.GetByName("graph"); +// index_blobs.emplace_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); +// continue; +// } +// reader(&len, sizeof(size_t), 1); +// assert(len != 0); +// auto binary = new uint8_t[len]; +// reader(binary, len, 1); +// index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); +// len = 0; +// } +// reader(&len, sizeof(size_t), 1); +// assert(len != 0); +// auto config = new char[len]; +// reader(config, len, 1); +// std::string index_config = config; +// delete[] config; + index_ptr_->LoadIndex(index_config, index_blobs); + } + +// PreprocessorPtr +// CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { +// return std::make_shared(); +//} + + IndexModelPtr + CPUSPTAGRNG::Train(const DatasetPtr& origin, const Config& train_config) { + SetParameters(train_config); + DatasetPtr dataset = origin->Clone(); + + // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine + // && preprocessor_) { + // preprocessor_->Preprocess(dataset); + //} + + auto vectorset = ConvertToVectorSet(dataset); + auto metaset = ConvertToMetadataSet(dataset); + index_ptr_->BuildIndex(vectorset, metaset); + + // TODO: return IndexModelPtr + return nullptr; + } + + void + CPUSPTAGRNG::Add(const DatasetPtr& origin, const Config& add_config) { + // SetParameters(add_config); + // DatasetPtr dataset = origin->Clone(); + // + // // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine + // // && preprocessor_) { + // // preprocessor_->Preprocess(dataset); + // //} + // + // auto vectorset = ConvertToVectorSet(dataset); + // auto metaset = ConvertToMetadataSet(dataset); + // index_ptr_->AddIndex(vectorset, metaset); + } + + void + CPUSPTAGRNG::SetParameters(const Config& config) { +#define Assign(param_name, str_name) \ + conf->param_name == INVALID_VALUE ? index_ptr_->SetParameter(str_name, std::to_string(build_cfg->param_name)) \ + : index_ptr_->SetParameter(str_name, std::to_string(conf->param_name)) + + if (index_type_ == SPTAG::IndexAlgoType::KDT) { + auto conf = std::dynamic_pointer_cast(config); + auto build_cfg = SPTAGParameterMgr::GetInstance().GetKDTParameters(); + + Assign(kdtnumber, "KDTNumber"); + Assign(numtopdimensionkdtsplit, "NumTopDimensionKDTSplit"); + Assign(samples, "Samples"); + Assign(tptnumber, "TPTNumber"); + Assign(tptleafsize, "TPTLeafSize"); + Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); + Assign(neighborhoodsize, "NeighborhoodSize"); + Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); + Assign(graphcefscale, "GraphCEFScale"); + Assign(refineiterations, "RefineIterations"); + Assign(cef, "CEF"); + Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); + Assign(numofthreads, "NumberOfThreads"); + Assign(maxcheck, "MaxCheck"); + Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); + Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); + Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); + } else { + auto conf = std::dynamic_pointer_cast(config); + auto build_cfg = SPTAGParameterMgr::GetInstance().GetBKTParameters(); + + Assign(bktnumber, "BKTNumber"); + Assign(bktkmeansk, "BKTKMeansK"); + Assign(bktleafsize, "BKTLeafSize"); + Assign(samples, "Samples"); + Assign(tptnumber, "TPTNumber"); + Assign(tptleafsize, "TPTLeafSize"); + Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); + Assign(neighborhoodsize, "NeighborhoodSize"); + Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); + Assign(graphcefscale, "GraphCEFScale"); + Assign(refineiterations, "RefineIterations"); + Assign(cef, "CEF"); + Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); + Assign(numofthreads, "NumberOfThreads"); + Assign(maxcheck, "MaxCheck"); + Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); + Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); + Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); + } + } + + DatasetPtr + CPUSPTAGRNG::Search(const DatasetPtr& dataset, const Config& config) { + SetParameters(config); + auto tensor = dataset->tensor()[0]; + auto p = (float*)tensor->raw_mutable_data(); + for (auto i = 0; i < 10; ++i) { + for (auto j = 0; j < 10; ++j) { + std::cout << p[i * 10 + j] << " "; + } + std::cout << std::endl; + } + std::vector query_results = ConvertToQueryResult(dataset, config); + +#pragma omp parallel for + for (auto i = 0; i < query_results.size(); ++i) { + auto target = (float*)query_results[i].GetTarget(); + std::cout << target[0] << ", " << target[1] << ", " << target[2] << std::endl; + index_ptr_->SearchIndex(query_results[i]); + } + + return ConvertToDataset(query_results); + } + + int64_t + CPUSPTAGRNG::Count() { + return index_ptr_->GetNumSamples(); + } + + int64_t + CPUSPTAGRNG::Dimension() { + return index_ptr_->GetFeatureDim(); + } + + VectorIndexPtr + CPUSPTAGRNG::Clone() { + KNOWHERE_THROW_MSG("not support"); + } + + void + CPUSPTAGRNG::Seal() { + return; // do nothing + } + + BinarySet + CPUSPTAGRNGIndexModel::Serialize() { + // KNOWHERE_THROW_MSG("not support"); // not support + } + + void + CPUSPTAGRNGIndexModel::Load(const BinarySet& binary) { + // KNOWHERE_THROW_MSG("not support"); // not support + } + +} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h new file mode 100644 index 0000000000..c78807a3ee --- /dev/null +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "VectorIndex.h" +#include "knowhere/index/IndexModel.h" + +namespace knowhere { + + class CPUSPTAGRNG : public VectorIndex { + public: + explicit CPUSPTAGRNG(const std::string& IndexType); + + public: + BinarySet + Serialize() override; + + VectorIndexPtr + Clone() override; + + void + Load(const BinarySet& index_array) override; + + public: + // PreprocessorPtr + // BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override; + + int64_t + Count() override; + + int64_t + Dimension() override; + + IndexModelPtr + Train(const DatasetPtr& dataset, const Config& config) override; + + void + Add(const DatasetPtr& dataset, const Config& config) override; + + DatasetPtr + Search(const DatasetPtr& dataset, const Config& config) override; + + void + Seal() override; + + private: + void + SetParameters(const Config& config); + + private: + PreprocessorPtr preprocessor_; + std::shared_ptr index_ptr_; + SPTAG::IndexAlgoType index_type_; + }; + + using CPUSPTAGRNGPtr = std::shared_ptr; + + class CPUSPTAGRNGIndexModel : public IndexModel { + public: + BinarySet + Serialize() override; + + void + Load(const BinarySet& binary) override; + + private: + std::shared_ptr index_; + }; + + using CPUSPTAGRNGIndexModelPtr = std::shared_ptr; + +} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h index b2854abef8..9bb3cac303 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -42,6 +42,32 @@ constexpr int64_t DEFAULT_OUT_DEGREE = INVALID_VALUE; constexpr int64_t DEFAULT_CANDIDATE_SISE = INVALID_VALUE; constexpr int64_t DEFAULT_NNG_K = INVALID_VALUE; +// SPTAG Config +constexpr int64_t DEFAULT_SAMPLES = INVALID_VALUE; +constexpr int64_t DEFAULT_TPTNUMBER = INVALID_VALUE; +constexpr int64_t DEFAULT_TPTLEAFSIZE = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMTOPDIMENSIONTPTSPLIT = INVALID_VALUE; +constexpr int64_t DEFAULT_NEIGHBORHOODSIZE = INVALID_VALUE; +constexpr int64_t DEFAULT_GRAPHNEIGHBORHOODSCALE = INVALID_VALUE; +constexpr int64_t DEFAULT_GRAPHCEFSCALE = INVALID_VALUE; +constexpr int64_t DEFAULT_REFINEITERATIONS = INVALID_VALUE; +constexpr int64_t DEFAULT_CEF = INVALID_VALUE; +constexpr int64_t DEFAULT_MAXCHECKFORREFINEGRAPH = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMOFTHREADS = INVALID_VALUE; +constexpr int64_t DEFAULT_MAXCHECK = INVALID_VALUE; +constexpr int64_t DEFAULT_THRESHOLDOFNUMBEROFCONTINUOUSNOBETTERPROPAGATION = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMBEROFINITIALDYNAMICPIVOTS = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMBEROFOTHERDYNAMICPIVOTS = INVALID_VALUE; + +// KDT Config +constexpr int64_t DEFAULT_KDTNUMBER = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMTOPDIMENSIONKDTSPLIT = INVALID_VALUE; + +// BKT Config +constexpr int64_t DEFAULT_BKTNUMBER = INVALID_VALUE; +constexpr int64_t DEFAULT_BKTKMEANSK = INVALID_VALUE; +constexpr int64_t DEFAULT_BKTLEAFSIZE = INVALID_VALUE; + struct IVFCfg : public Cfg { int64_t nlist = DEFAULT_NLIST; int64_t nprobe = DEFAULT_NPROBE; @@ -126,8 +152,57 @@ struct NSGCfg : public IVFCfg { }; using NSGConfig = std::shared_ptr; -struct KDTCfg : public Cfg { - int64_t tptnubmber = -1; +struct SPTAGCfg : public Cfg { + int64_t samples = DEFAULT_SAMPLES; + int64_t tptnumber = DEFAULT_TPTNUMBER; + int64_t tptleafsize = DEFAULT_TPTLEAFSIZE; + int64_t numtopdimensiontptsplit = DEFAULT_NUMTOPDIMENSIONTPTSPLIT; + int64_t neighborhoodsize = DEFAULT_NEIGHBORHOODSIZE; + int64_t graphneighborhoodscale = DEFAULT_GRAPHNEIGHBORHOODSCALE; + int64_t graphcefscale = DEFAULT_GRAPHCEFSCALE; + int64_t refineiterations = DEFAULT_REFINEITERATIONS; + int64_t cef = DEFAULT_CEF; + int64_t maxcheckforrefinegraph = DEFAULT_MAXCHECKFORREFINEGRAPH; + int64_t numofthreads = DEFAULT_NUMOFTHREADS; + int64_t maxcheck = DEFAULT_MAXCHECK; + int64_t thresholdofnumberofcontinuousnobetterpropagation = DEFAULT_THRESHOLDOFNUMBEROFCONTINUOUSNOBETTERPROPAGATION; + int64_t numberofinitialdynamicpivots = DEFAULT_NUMBEROFINITIALDYNAMICPIVOTS; + int64_t numberofotherdynamicpivots = DEFAULT_NUMBEROFOTHERDYNAMICPIVOTS; + + SPTAGCfg() = default; + + bool + CheckValid() override { + return true; + }; }; +using SPTAGConfig = std::shared_ptr; + +struct KDTCfg : public SPTAGCfg { + int64_t kdtnumber = DEFAULT_KDTNUMBER; + int64_t numtopdimensionkdtsplit = DEFAULT_NUMTOPDIMENSIONKDTSPLIT; + + KDTCfg() = default; + + bool + CheckValid() override { + return true; + }; +}; +using KDTConfig = std::shared_ptr; + +struct BKTCfg : public SPTAGCfg { + int64_t bktnumber = DEFAULT_BKTNUMBER; + int64_t bktkmeansk = DEFAULT_BKTKMEANSK; + int64_t bktleafsize = DEFAULT_BKTLEAFSIZE; + + BKTCfg() = default; + + bool + CheckValid() override { + return true; + }; +}; +using BKTConfig = std::shared_ptr; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp deleted file mode 100644 index 19bf070dba..0000000000 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h" - -namespace knowhere { - -const std::vector& -KDTParameterMgr::GetKDTParameters() { - return kdt_parameters_; -} - -KDTParameterMgr::KDTParameterMgr() { - kdt_parameters_ = std::vector{ - {"KDTNumber", "1"}, - {"NumTopDimensionKDTSplit", "5"}, - {"NumSamplesKDTSplitConsideration", "100"}, - - {"TPTNumber", "1"}, - {"TPTLeafSize", "2000"}, - {"NumTopDimensionTPTSplit", "5"}, - - {"NeighborhoodSize", "32"}, - {"GraphNeighborhoodScale", "2"}, - {"GraphCEFScale", "2"}, - {"RefineIterations", "0"}, - {"CEF", "1000"}, - {"MaxCheckForRefineGraph", "10000"}, - - {"NumberOfThreads", "1"}, - - {"MaxCheck", "8192"}, - {"ThresholdOfNumberOfContinuousNoBetterPropagation", "3"}, - {"NumberOfInitialDynamicPivots", "50"}, - {"NumberOfOtherDynamicPivots", "4"}, - }; -} - -} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp new file mode 100644 index 0000000000..f6e1409abd --- /dev/null +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "knowhere/index/vector_index/helpers/SPTAGParameterMgr.h" + +namespace knowhere { + + const KDTConfig& + SPTAGParameterMgr::GetKDTParameters() { + return kdt_config_; + } + + const BKTConfig& + SPTAGParameterMgr::GetBKTParameters() { + return bkt_config_; + } + + SPTAGParameterMgr::SPTAGParameterMgr() { + kdt_config_ = std::make_shared(); + kdt_config_->kdtnumber = 1; + kdt_config_->numtopdimensionkdtsplit = 5; + kdt_config_->samples = 100; + kdt_config_->tptnumber = 1; + kdt_config_->tptleafsize = 2000; + kdt_config_->numtopdimensiontptsplit = 5; + kdt_config_->neighborhoodsize = 32; + kdt_config_->graphneighborhoodscale = 2; + kdt_config_->graphcefscale = 2; + kdt_config_->refineiterations = 0; + kdt_config_->cef = 1000; + kdt_config_->maxcheckforrefinegraph = 10000; + kdt_config_->numofthreads = 1; + kdt_config_->maxcheck = 8192; + kdt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; + kdt_config_->numberofinitialdynamicpivots = 50; + kdt_config_->numberofotherdynamicpivots = 4; + + bkt_config_ = std::make_shared(); + bkt_config_->bktnumber = 1; + bkt_config_->bktkmeansk = 32; + bkt_config_->bktleafsize = 8; + bkt_config_->samples = 100; + bkt_config_->tptnumber = 1; + bkt_config_->tptleafsize = 2000; + bkt_config_->numtopdimensiontptsplit = 5; + bkt_config_->neighborhoodsize = 32; + bkt_config_->graphneighborhoodscale = 2; + bkt_config_->graphcefscale = 2; + bkt_config_->refineiterations = 0; + bkt_config_->cef = 1000; + bkt_config_->maxcheckforrefinegraph = 10000; + bkt_config_->numofthreads = 1; + bkt_config_->maxcheck = 8192; + bkt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; + bkt_config_->numberofinitialdynamicpivots = 50; + bkt_config_->numberofotherdynamicpivots = 4; + } + +} // namespace knowhere \ No newline at end of file diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h similarity index 55% rename from core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h rename to core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h index fe90761e17..d215d9e275 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h @@ -22,31 +22,40 @@ #include #include +#include +#include "IndexParameter.h" + namespace knowhere { -using KDTParameter = std::pair; + using KDTConfig = std::shared_ptr; + using BKTConfig = std::shared_ptr; -class KDTParameterMgr { - public: - const std::vector& - GetKDTParameters(); + class SPTAGParameterMgr { + public: + const KDTConfig& + GetKDTParameters(); - public: - static KDTParameterMgr& - GetInstance() { - static KDTParameterMgr instance; - return instance; - } + const BKTConfig& + GetBKTParameters(); - KDTParameterMgr(const KDTParameterMgr&) = delete; - KDTParameterMgr& - operator=(const KDTParameterMgr&) = delete; + public: + static SPTAGParameterMgr& + GetInstance() { + static SPTAGParameterMgr instance; + return instance; + } - private: - KDTParameterMgr(); + SPTAGParameterMgr(const SPTAGParameterMgr&) = delete; - private: - std::vector kdt_parameters_; -}; + SPTAGParameterMgr& + operator=(const SPTAGParameterMgr&) = delete; + + private: + SPTAGParameterMgr(); + + private: + KDTConfig kdt_config_; + BKTConfig bkt_config_; + }; } // namespace knowhere diff --git a/core/src/index/thirdparty/SPTAG/AnnService/inc/Core/Common/Dataset.h b/core/src/index/thirdparty/SPTAG/AnnService/inc/Core/Common/Dataset.h index d00ea45365..0208f6d983 100644 --- a/core/src/index/thirdparty/SPTAG/AnnService/inc/Core/Common/Dataset.h +++ b/core/src/index/thirdparty/SPTAG/AnnService/inc/Core/Common/Dataset.h @@ -195,7 +195,7 @@ namespace SPTAG C = *((DimensionType*)pDataPointsMemFile); pDataPointsMemFile += sizeof(DimensionType); - Initialize(R, C, (T*)pDataPointsMemFile); + Initialize(R, C, (T*)pDataPointsMemFile, false); std::cout << "Load " << name << " (" << R << ", " << C << ") Finish!" << std::endl; return true; } diff --git a/core/src/index/unittest/CMakeLists.txt b/core/src/index/unittest/CMakeLists.txt index d3ba84e557..93ae63a9ec 100644 --- a/core/src/index/unittest/CMakeLists.txt +++ b/core/src/index/unittest/CMakeLists.txt @@ -82,17 +82,17 @@ if (NOT TARGET test_idmap) endif () target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) -# -set(kdt_srcs +# +set(sptag_srcs ${INDEX_SOURCE_DIR}/knowhere/knowhere/adapter/SptagAdapter.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/preprocessor/Normalize.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexKDT.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp ) -if (NOT TARGET test_kdt) - add_executable(test_kdt test_kdt.cpp ${kdt_srcs} ${util_srcs}) +if (NOT TARGET test_sptag) + add_executable(test_sptag test_sptag.cpp ${sptag_srcs} ${util_srcs}) endif () -target_link_libraries(test_kdt +target_link_libraries(test_sptag SPTAGLibStatic ${depend_libs} ${unittest_libs} ${basic_libs}) @@ -106,7 +106,7 @@ endif () install(TARGETS test_ivf DESTINATION unittest) install(TARGETS test_idmap DESTINATION unittest) -install(TARGETS test_kdt DESTINATION unittest) +install(TARGETS test_sptag DESTINATION unittest) if (KNOWHERE_GPU_VERSION) install(TARGETS test_gpuresource DESTINATION unittest) install(TARGETS test_customized_index DESTINATION unittest) diff --git a/core/src/index/unittest/test_kdt.cpp b/core/src/index/unittest/test_kdt.cpp deleted file mode 100644 index bbc7dcf94c..0000000000 --- a/core/src/index/unittest/test_kdt.cpp +++ /dev/null @@ -1,144 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include -#include - -#include "knowhere/adapter/SptagAdapter.h" -#include "knowhere/adapter/Structure.h" -#include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/IndexKDT.h" -#include "knowhere/index/vector_index/helpers/Definitions.h" - -#include "unittest/utils.h" - -using ::testing::Combine; -using ::testing::TestWithParam; -using ::testing::Values; - -class KDTTest : public DataGen, public ::testing::Test { - protected: - void - SetUp() override { - Generate(96, 1000, 10); - index_ = std::make_shared(); - - auto tempconf = std::make_shared(); - tempconf->tptnubmber = 1; - tempconf->k = 10; - conf = tempconf; - - Init_with_default(); - } - - protected: - knowhere::Config conf; - std::shared_ptr index_ = nullptr; -}; - -// TODO(lxj): add test about count() and dimension() -TEST_F(KDTTest, kdt_basic) { - assert(!xb.empty()); - - auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); - index_->set_preprocessor(preprocessor); - - auto model = index_->Train(base_dataset, conf); - index_->set_index_model(model); - index_->Add(base_dataset, conf); - auto result = index_->Search(query_dataset, conf); - AssertAnns(result, nq, k); - - { - // auto ids = result->array()[0]; - // auto dists = result->array()[1]; - auto ids = result->ids(); - auto dists = result->dist(); - - std::stringstream ss_id; - std::stringstream ss_dist; - for (auto i = 0; i < nq; i++) { - for (auto j = 0; j < k; ++j) { - ss_id << *((int64_t*)(ids) + i * k + j) << " "; - ss_dist << *((float*)(dists) + i * k + j) << " "; - // ss_id << *ids->data()->GetValues(1, i * k + j) << " "; - // ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; - } - ss_id << std::endl; - ss_dist << std::endl; - } - std::cout << "id\n" << ss_id.str() << std::endl; - std::cout << "dist\n" << ss_dist.str() << std::endl; - } -} - -// TODO(zirui): enable test -// TEST_F(KDTTest, kdt_serialize) { -// assert(!xb.empty()); -// -// auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); -// index_->set_preprocessor(preprocessor); -// -// auto model = index_->Train(base_dataset, conf); -// // index_->Add(base_dataset, conf); -// auto binaryset = index_->Serialize(); -// auto new_index = std::make_shared(); -// new_index->Load(binaryset); -// auto result = new_index->Search(query_dataset, conf); -// AssertAnns(result, nq, k); -// PrintResult(result, nq, k); -// ASSERT_EQ(new_index->Count(), nb); -// ASSERT_EQ(new_index->Dimension(), dim); -// ASSERT_THROW({ new_index->Clone(); }, knowhere::KnowhereException); -// ASSERT_NO_THROW({ new_index->Seal(); }); -// -// { -// int fileno = 0; -// const std::string& base_name = "/tmp/kdt_serialize_test_bin_"; -// std::vector filename_list; -// std::vector> meta_list; -// for (auto& iter : binaryset.binary_map_) { -// const std::string& filename = base_name + std::to_string(fileno); -// FileIOWriter writer(filename); -// writer(iter.second->data.get(), iter.second->size); -// -// meta_list.emplace_back(std::make_pair(iter.first, iter.second->size)); -// filename_list.push_back(filename); -// ++fileno; -// } -// -// knowhere::BinarySet load_data_list; -// for (int i = 0; i < filename_list.size() && i < meta_list.size(); ++i) { -// auto bin_size = meta_list[i].second; -// FileIOReader reader(filename_list[i]); -// -// auto load_data = new uint8_t[bin_size]; -// reader(load_data, bin_size); -// auto data = std::make_shared(); -// data.reset(load_data); -// load_data_list.Append(meta_list[i].first, data, bin_size); -// } -// -// auto new_index = std::make_shared(); -// new_index->Load(load_data_list); -// auto result = new_index->Search(query_dataset, conf); -// AssertAnns(result, nq, k); -// PrintResult(result, nq, k); -// } -//} diff --git a/core/src/index/unittest/test_sptag.cpp b/core/src/index/unittest/test_sptag.cpp new file mode 100644 index 0000000000..a0898a619b --- /dev/null +++ b/core/src/index/unittest/test_sptag.cpp @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#include "knowhere/adapter/SptagAdapter.h" +#include "knowhere/adapter/Structure.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexSPTAG.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" + +#include "unittest/utils.h" + +using ::testing::Combine; +using ::testing::TestWithParam; +using ::testing::Values; + +class SPTAGTest : public DataGen, public TestWithParam { + protected: + void + SetUp() override { + IndexType = GetParam(); + Generate(128, 100, 5); + index_ = std::make_shared(IndexType); + if (IndexType == "KDT") { + auto tempconf = std::make_shared(); + tempconf->tptnumber = 1; + tempconf->k = 10; + conf = tempconf; + } else { + auto tempconf = std::make_shared(); + tempconf->tptnumber = 1; + tempconf->k = 10; + conf = tempconf; + } + + Init_with_default(); + } + + protected: + knowhere::Config conf; + std::shared_ptr index_ = nullptr; + std::string IndexType; +}; + +INSTANTIATE_TEST_CASE_P(SPTAGParameters, SPTAGTest, Values("KDT", "BKT")); + +// TODO(lxj): add test about count() and dimension() +TEST_P(SPTAGTest, sptag_basic) { + assert(!xb.empty()); + + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); + index_->set_preprocessor(preprocessor); + + auto model = index_->Train(base_dataset, conf); + index_->set_index_model(model); + index_->Add(base_dataset, conf); + auto result = index_->Search(query_dataset, conf); + AssertAnns(result, nq, k); + + { + //auto ids = result->array()[0]; + //auto dists = result->array()[1]; + auto ids = result->ids(); + auto dists = result->dist(); + + std::stringstream ss_id; + std::stringstream ss_dist; + for (auto i = 0; i < nq; i++) { + for (auto j = 0; j < k; ++j) { + //ss_id << *ids->data()->GetValues(1, i * k + j) << " "; + //ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; + ss_id << *((int64_t*)(ids) + i * k + j) << " "; + ss_dist << *((float*)(dists) + i * k + j) << " "; + } + ss_id << std::endl; + ss_dist << std::endl; + } + std::cout << "id\n" << ss_id.str() << std::endl; + std::cout << "dist\n" << ss_dist.str() << std::endl; + } +} + +TEST_P(SPTAGTest, sptag_serialize) { + assert(!xb.empty()); + + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); + index_->set_preprocessor(preprocessor); + + auto model = index_->Train(base_dataset, conf); + + index_->Add(base_dataset, conf); + auto binaryset = index_->Serialize(); + auto new_index = std::make_shared(IndexType); + new_index->Load(binaryset); + auto result = new_index->Search(query_dataset, conf); + AssertAnns(result, nq, k); + PrintResult(result, nq, k); + ASSERT_EQ(new_index->Count(), nb); + ASSERT_EQ(new_index->Dimension(), dim); + // ASSERT_THROW({ new_index->Clone(); }, knowhere::KnowhereException); + // ASSERT_NO_THROW({ new_index->Seal(); }); + + { + int fileno = 0; + const std::string& base_name = "/tmp/sptag_serialize_test_bin_"; + std::vector filename_list; + std::vector> meta_list; + for (auto& iter : binaryset.binary_map_) { + const std::string& filename = base_name + std::to_string(fileno); + FileIOWriter writer(filename); + writer(iter.second->data.get(), iter.second->size); + + meta_list.emplace_back(std::make_pair(iter.first, iter.second->size)); + filename_list.push_back(filename); + ++fileno; + } + + knowhere::BinarySet load_data_list; + for (int i = 0; i < filename_list.size() && i < meta_list.size(); ++i) { + auto bin_size = meta_list[i].second; + FileIOReader reader(filename_list[i]); + + auto load_data = new uint8_t[bin_size]; + reader(load_data, bin_size); + auto data = std::make_shared(); + data.reset(load_data); + load_data_list.Append(meta_list[i].first, data, bin_size); + } + + auto new_index = std::make_shared(IndexType); + new_index->Load(load_data_list); + auto result = new_index->Search(query_dataset, conf); + AssertAnns(result, nq, k); + PrintResult(result, nq, k); + } +} diff --git a/core/src/index/unittest/utils.cpp b/core/src/index/unittest/utils.cpp index 2556b60fad..110d8fbf92 100644 --- a/core/src/index/unittest/utils.cpp +++ b/core/src/index/unittest/utils.cpp @@ -153,22 +153,24 @@ void AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->ids(); for (auto i = 0; i < nq; i++) { - EXPECT_EQ(i, *((int64_t*)(ids) + i * k)); + EXPECT_EQ(i, *((int64_t*)(ids) + i * k)); // EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); } } void PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) { - auto ids = result->array()[0]; - auto dists = result->array()[1]; + auto ids = result->ids(); + auto dists = result->dist(); std::stringstream ss_id; std::stringstream ss_dist; - for (auto i = 0; i < 10; i++) { + for (auto i = 0; i < nq; i++) { for (auto j = 0; j < k; ++j) { - ss_id << *(ids->data()->GetValues(1, i * k + j)) << " "; - ss_dist << *(dists->data()->GetValues(1, i * k + j)) << " "; + //ss_id << *(ids->data()->GetValues(1, i * k + j)) << " "; + //ss_dist << *(dists->data()->GetValues(1, i * k + j)) << " "; + ss_id << *((int64_t*)(ids) + i * k + j) << " "; + ss_dist << *((float*)(dists) + i * k + j) << " "; } ss_id << std::endl; ss_dist << std::endl; diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 461745f1fd..4114eb2af4 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -204,5 +204,35 @@ NSGConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) return conf; } +knowhere::Config +SPTAGKDTConfAdapter::Match(const TempMetaConf& metaconf) { + auto conf = std::make_shared(); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + return conf; +} + +knowhere::Config +SPTAGKDTConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; + return conf; +} + +knowhere::Config +SPTAGBKTConfAdapter::Match(const TempMetaConf& metaconf) { + auto conf = std::make_shared(); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + return conf; +} + +knowhere::Config +SPTAGBKTConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; + return conf; +} + } // namespace engine } // namespace milvus diff --git a/core/src/wrapper/ConfAdapter.h b/core/src/wrapper/ConfAdapter.h index 5ec3d52486..600dd3a517 100644 --- a/core/src/wrapper/ConfAdapter.h +++ b/core/src/wrapper/ConfAdapter.h @@ -97,5 +97,23 @@ class NSGConfAdapter : public IVFConfAdapter { MatchSearch(const TempMetaConf& metaconf, const IndexType& type) final; }; +class SPTAGKDTConfAdapter : public ConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf& metaconf) override; + + knowhere::Config + MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; +}; + +class SPTAGBKTConfAdapter : public ConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf& metaconf) override; + + knowhere::Config + MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; +}; + } // namespace engine } // namespace milvus diff --git a/core/src/wrapper/ConfAdapterMgr.cpp b/core/src/wrapper/ConfAdapterMgr.cpp index d0eba04529..cf58c0110f 100644 --- a/core/src/wrapper/ConfAdapterMgr.cpp +++ b/core/src/wrapper/ConfAdapterMgr.cpp @@ -56,6 +56,9 @@ AdapterMgr::RegisterAdapter() { REGISTER_CONF_ADAPTER(IVFPQConfAdapter, IndexType::FAISS_IVFPQ_MIX, ivfpq_mix); REGISTER_CONF_ADAPTER(NSGConfAdapter, IndexType::NSG_MIX, nsg_mix); + + REGISTER_CONF_ADAPTER(SPTAGKDTConfAdapter, IndexType::SPTAG_KDT_RNT_CPU, sptag_kdt); + REGISTER_CONF_ADAPTER(SPTAGBKTConfAdapter, IndexType::SPTAG_BKT_RNT_CPU, sptag_bkt); } } // namespace engine diff --git a/core/src/wrapper/VecIndex.cpp b/core/src/wrapper/VecIndex.cpp index 81a13c60f9..9f68cac0a1 100644 --- a/core/src/wrapper/VecIndex.cpp +++ b/core/src/wrapper/VecIndex.cpp @@ -22,7 +22,7 @@ #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/IndexIVFPQ.h" #include "knowhere/index/vector_index/IndexIVFSQ.h" -#include "knowhere/index/vector_index/IndexKDT.h" +#include "knowhere/index/vector_index/IndexSPTAG.h" #include "knowhere/index/vector_index/IndexNSG.h" #include "utils/Log.h" @@ -128,7 +128,11 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) { break; } case IndexType::SPTAG_KDT_RNT_CPU: { - index = std::make_shared(); + index = std::make_shared("KDT"); + break; + } + case IndexType::SPTAG_BKT_RNT_CPU: { + index = std::make_shared("BKT"); break; } case IndexType::FAISS_IVFSQ8_CPU: { diff --git a/core/src/wrapper/VecIndex.h b/core/src/wrapper/VecIndex.h index efe01a25d7..e69655b087 100644 --- a/core/src/wrapper/VecIndex.h +++ b/core/src/wrapper/VecIndex.h @@ -49,6 +49,7 @@ enum class IndexType { FAISS_IVFSQ8_HYBRID, // only support build on gpu. NSG_MIX, FAISS_IVFPQ_MIX, + SPTAG_BKT_RNT_CPU, }; class VecIndex; @@ -139,6 +140,9 @@ write_index(VecIndexPtr index, const std::string& location); extern VecIndexPtr read_index(const std::string& location); +VecIndexPtr +read_index(const std::string& location, knowhere::BinarySet& index_binary); + extern VecIndexPtr GetVecIndexFactory(const IndexType& type, const Config& cfg = Config()); diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 025601a1cd..413db60e37 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -29,15 +29,16 @@ INITIALIZE_EASYLOGGINGPP +using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; -using ::testing::Combine; class KnowhereWrapperTest - : public DataGenBase, - public TestWithParam<::std::tuple> { + : public DataGenBase, + public TestWithParam<::std::tuple> { protected: - void SetUp() override { + void + SetUp() override { #ifdef MILVUS_GPU_VERSION knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, PINMEM, TEMPMEM, RESNUM); #endif @@ -57,12 +58,13 @@ class KnowhereWrapperTest conf = ParamGenerator::GetInstance().GenBuild(index_type, tempconf); searchconf = ParamGenerator::GetInstance().GenSearchConf(index_type, tempconf); -// conf->k = k; -// conf->d = dim; -// conf->gpu_id = DEVICEID; + // conf->k = k; + // conf->d = dim; + // conf->gpu_id = DEVICEID; } - void TearDown() override { + void + TearDown() override { #ifdef MILVUS_GPU_VERSION knowhere::FaissGpuResourceMgr::GetInstance().Free(); #endif @@ -75,22 +77,21 @@ class KnowhereWrapperTest knowhere::Config searchconf; }; -INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest, - Values( - //["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"] +INSTANTIATE_TEST_CASE_P( + WrapperParam, KnowhereWrapperTest, + Values( +//["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"] #ifdef MILVUS_GPU_VERSION std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 1000, 10, 10), - // std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, - // 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFPQ_MIX, "Default", 64, 1000, 10, 10), - -// std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), + // std::make_tuple(milvus::engine::IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), #endif - // std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10), + // std::make_tuple(milvus::engine::IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 100, 10, 10), + // std::make_tuple(milvus::engine::IndexType::SPTAG_BKT_RNT_CPU, "Default", 128, 100, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IDMAP, "Default", 64, 1000, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 1000, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10))); From 84c1483d969ea3d3d8c5f64f2d575271d4d2d1aa Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 16:10:36 +0800 Subject: [PATCH 57/82] [skip ci] Update README_JP --- README_JP.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README_JP.md b/README_JP.md index fd80b5d2ca..d55fea4a14 100644 --- a/README_JP.md +++ b/README_JP.md @@ -5,7 +5,7 @@ ![LICENSE](https://img.shields.io/badge/license-Apache--2.0-brightgreen) ![Language](https://img.shields.io/badge/language-C%2B%2B-blue) [![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master) -![Release](https://img.shields.io/badge/release-v0.5.2-yellowgreen) +![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen) ![Release_date](https://img.shields.io/badge/release%20date-November-yellowgreen) @@ -15,9 +15,9 @@ Milvusは世界中一番早い特徴ベクトルにむかう類似性検索エンジンです。不均質な計算アーキテクチャーに基づいて効率を最大化出来ます。数十億のベクタの中に目標を検索できるまで数ミリ秒しかかからず、最低限の計算資源だけが必要です。 -Milvusは安定的なPython、Java又は C++ APIsを提供します。 +Milvusは安定的な[Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java)又は [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) APIsを提供します。 -Milvus [リリースノート](https://milvus.io/docs/en/release/v0.5.2/)を読んで最新バージョンや更新情報を手に入れます。 +Milvus [リリースノート](https://milvus.io/docs/en/release/v0.5.3/)を読んで最新バージョンや更新情報を手に入れます。(https://github.com/milvus-io/milvus/tree/master/core/src/sdk) ## はじめに @@ -46,7 +46,7 @@ C++サンプルコードを実行するために、次のコマンドをつか 本プロジェクトへの貢献に心より感謝いたします。 Milvusを貢献したいと思うなら、[貢献規約](CONTRIBUTING.md)を読んでください。 本プロジェクトはMilvusの[行動規範](CODE_OF_CONDUCT.md)に従います。プロジェクトに参加したい場合は、行動規範を従ってください。 -[GitHub issues](https://github.com/milvus-io/milvus/issues/new/choose) を使って問題やバッグなとを報告しでください。 一般てきな問題なら, Milvusコミュニティに参加してください。 +[GitHub issues](https://github.com/milvus-io/milvus/issues) を使って問題やバッグなとを報告しでください。 一般てきな問題なら, Milvusコミュニティに参加してください。 ## Milvusコミュニティを参加する @@ -59,6 +59,8 @@ C++サンプルコードを実行するために、次のコマンドをつか - [Milvus](https://github.com/milvus-io/bootcamp) +- [Milvus テストレポート](https://github.com/milvus-io/milvus/tree/master/docs/test_report) + - [Milvus Medium](https://medium.com/@milvusio) - [Milvus CSDN](https://zilliz.blog.csdn.net/) From a54df0d3ccf06c0b36df9ada37919016cce2a95c Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 16:57:22 +0800 Subject: [PATCH 58/82] [skip ci] minor change --- README_CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_CN.md b/README_CN.md index 979c476ebf..df407f1a5f 100644 --- a/README_CN.md +++ b/README_CN.md @@ -65,7 +65,7 @@ Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java]( - [Milvus 在线训练营](https://github.com/milvus-io/bootcamp) -- [Milvus 测试报告](https://github.com/milvus-io/milvus/tree/master/docs/test_report_cn) +- [Milvus 测试报告](https://github.com/milvus-io/milvus/tree/master/docs/test_report) - [Milvus Medium](https://medium.com/@milvusio) From da0b9e05ac8e341f8533073deabd9a4dadd355d8 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 17:07:20 +0800 Subject: [PATCH 59/82] [skip ci] Add test report list --- docs/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/README.md b/docs/README.md index 64409d8240..7465fe839f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,3 +9,8 @@ The tests are run on [SIFT1B dataset](http://corpus-texmex.irisa.fr/), and provi - Recall: The fraction of the total amount of relevant instances that were actually retrieved. Test variables are `nq` and `topk`. + +The following is a list of existing test reports: + +- [IVF_SQ8](test_report/milvus_ivfsq8_test_report_detailed_version.md) +- [IVF_SQ8H](test_report/milvus_ivfsq8h_test_report_detailed_version.md) From 323e9cfd79cd8444b77cca602d8700f2590818ff Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 17:12:42 +0800 Subject: [PATCH 60/82] [skip ci] Add cn version links --- docs/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/README.md b/docs/README.md index 7465fe839f..d57bbabaf2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,3 +14,8 @@ The following is a list of existing test reports: - [IVF_SQ8](test_report/milvus_ivfsq8_test_report_detailed_version.md) - [IVF_SQ8H](test_report/milvus_ivfsq8h_test_report_detailed_version.md) + +To read the CN version of these reports: + +- [IVF_SQ8_cn](test_report/milvus_ivfsq8_test_report_detailed_version_cn.md) +- [IVF_SQ8H_cn](test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md) From 24e05a87d09a3447e51bbb384800737ef2e93d2f Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 17:15:07 +0800 Subject: [PATCH 61/82] [skip ci] minor update --- docs/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/README.md b/docs/README.md index d57bbabaf2..f5f964de0d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,8 @@ The tests are run on [SIFT1B dataset](http://corpus-texmex.irisa.fr/), and provi Test variables are `nq` and `topk`. +## Test reports + The following is a list of existing test reports: - [IVF_SQ8](test_report/milvus_ivfsq8_test_report_detailed_version.md) From fd9c1a123065b4c5c4a28299ace4dd02aae88b20 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 17:21:10 +0800 Subject: [PATCH 62/82] [skip ci] Update test reports link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b2fc4454b..d8d5d80b11 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Below is a list of Milvus contributors. We greatly appreciate your contributions - [Milvus bootcamp](https://github.com/milvus-io/bootcamp) -- [Milvus test reports](https://github.com/milvus-io/milvus/tree/master/docs/test_report) +- [Milvus test reports](https://github.com/milvus-io/milvus/tree/master/docs) - [Milvus Medium](https://medium.com/@milvusio) From fd9beb8d3b173f4522bf0308637a98ae7a00172c Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 17:22:18 +0800 Subject: [PATCH 63/82] [skip ci] Update test report link --- README_CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_CN.md b/README_CN.md index df407f1a5f..df5445d931 100644 --- a/README_CN.md +++ b/README_CN.md @@ -65,7 +65,7 @@ Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java]( - [Milvus 在线训练营](https://github.com/milvus-io/bootcamp) -- [Milvus 测试报告](https://github.com/milvus-io/milvus/tree/master/docs/test_report) +- [Milvus 测试报告](https://github.com/milvus-io/milvus/tree/master/docs) - [Milvus Medium](https://medium.com/@milvusio) From e8f97ece6f047dac0059f1decb33d5bfe6f13a92 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 17:22:53 +0800 Subject: [PATCH 64/82] [skip ci] Update test report link --- README_JP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_JP.md b/README_JP.md index d55fea4a14..b5001a476c 100644 --- a/README_JP.md +++ b/README_JP.md @@ -59,7 +59,7 @@ C++サンプルコードを実行するために、次のコマンドをつか - [Milvus](https://github.com/milvus-io/bootcamp) -- [Milvus テストレポート](https://github.com/milvus-io/milvus/tree/master/docs/test_report) +- [Milvus テストレポート](https://github.com/milvus-io/milvus/tree/master/docs) - [Milvus Medium](https://medium.com/@milvusio) From 388e7fc315ca4e19b2e4269c419d39f8c970b615 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 17:23:47 +0800 Subject: [PATCH 65/82] [skip ci] minor delete --- README_JP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_JP.md b/README_JP.md index b5001a476c..4a1d67738d 100644 --- a/README_JP.md +++ b/README_JP.md @@ -17,7 +17,7 @@ Milvusは世界中一番早い特徴ベクトルにむかう類似性検索エ Milvusは安定的な[Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java)又は [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) APIsを提供します。 -Milvus [リリースノート](https://milvus.io/docs/en/release/v0.5.3/)を読んで最新バージョンや更新情報を手に入れます。(https://github.com/milvus-io/milvus/tree/master/core/src/sdk) +Milvus [リリースノート](https://milvus.io/docs/en/release/v0.5.3/)を読んで最新バージョンや更新情報を手に入れます。 ## はじめに From 7c9b627f2baea3b403263da30573a7e54dd47673 Mon Sep 17 00:00:00 2001 From: groot Date: Wed, 20 Nov 2019 17:54:45 +0800 Subject: [PATCH 66/82] #433 C++ SDK query result is not easy to use --- CHANGELOG.md | 1 + .../sdk/examples/partition/src/ClientTest.cpp | 1 + .../sdk/examples/simple/src/ClientTest.cpp | 1 + core/src/sdk/examples/utils/Utils.cpp | 35 +++++++++---------- core/src/sdk/grpc/ClientProxy.cpp | 17 +++++---- core/src/sdk/include/MilvusApi.h | 4 +-- 6 files changed, 33 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62c82df007..c9a93774cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#358 - Add more information in build.sh and install.md - \#404 - Add virtual method Init() in Pass abstract class - \#409 - Add a Fallback pass in optimizer +- \#433 - C++ SDK query result is not easy to use ## Task diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index 775e1f6d60..a12a7ff50e 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -148,6 +148,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { } { // wait unit build index finish + milvus_sdk::TimeRecorder rc("Create index"); std::cout << "Wait until create all index done" << std::endl; milvus::IndexParam index1 = BuildIndexParam(); milvus_sdk::Utils::PrintIndexParam(index1); diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp index dfa5e2219e..016c9eceac 100644 --- a/core/src/sdk/examples/simple/src/ClientTest.cpp +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -150,6 +150,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { } { // wait unit build index finish + milvus_sdk::TimeRecorder rc("Create index"); std::cout << "Wait until create all index done" << std::endl; milvus::IndexParam index1 = BuildIndexParam(); milvus_sdk::Utils::PrintIndexParam(index1); diff --git a/core/src/sdk/examples/utils/Utils.cpp b/core/src/sdk/examples/utils/Utils.cpp index da5e854e9b..fa373cd498 100644 --- a/core/src/sdk/examples/utils/Utils.cpp +++ b/core/src/sdk/examples/utils/Utils.cpp @@ -157,18 +157,20 @@ void Utils::PrintSearchResult(const std::vector>& search_record_array, const milvus::TopKQueryResult& topk_query_result) { BLOCK_SPLITER - size_t nq = topk_query_result.row_num; - size_t topk = topk_query_result.ids.size() / nq; - std::cout << "Returned result count: " << nq * topk << std::endl; + std::cout << "Returned result count: " << topk_query_result.size() << std::endl; - int32_t index = 0; - for (size_t i = 0; i < nq; i++) { - auto search_id = search_record_array[index].first; - index++; - std::cout << "No." << index << " vector " << search_id << " top " << topk << " search result:" << std::endl; + if (topk_query_result.size() != search_record_array.size()) { + std::cout << "ERROR: Returned result count dones equal nq" << std::endl; + return; + } + + for (size_t i = 0; i < topk_query_result.size(); i++) { + const milvus::QueryResult& one_result = topk_query_result[i]; + size_t topk = one_result.ids.size(); + auto search_id = search_record_array[i].first; + std::cout << "No." << i << " vector " << search_id << " top " << topk << " search result:" << std::endl; for (size_t j = 0; j < topk; j++) { - size_t idx = i * topk + j; - std::cout << "\t" << topk_query_result.ids[idx] << "\t" << topk_query_result.distances[idx] << std::endl; + std::cout << "\t" << one_result.ids[j] << "\t" << one_result.distances[j] << std::endl; } } BLOCK_SPLITER @@ -178,12 +180,11 @@ void Utils::CheckSearchResult(const std::vector>& search_record_array, const milvus::TopKQueryResult& topk_query_result) { BLOCK_SPLITER - size_t nq = topk_query_result.row_num; - size_t result_k = topk_query_result.ids.size() / nq; - int64_t index = 0; + size_t nq = topk_query_result.size(); for (size_t i = 0; i < nq; i++) { - auto result_id = topk_query_result.ids[i * result_k]; - auto search_id = search_record_array[index++].first; + const milvus::QueryResult& one_result = topk_query_result[i]; + auto search_id = search_record_array[i].first; + int64_t result_id = one_result.ids[0]; if (result_id != search_id) { std::cout << "The top 1 result is wrong: " << result_id << " vs. " << search_id << std::endl; } else { @@ -198,9 +199,7 @@ Utils::DoSearch(std::shared_ptr conn, const std::string& tab const std::vector& partiton_tags, int64_t top_k, int64_t nprobe, const std::vector>& search_record_array, milvus::TopKQueryResult& topk_query_result) { - topk_query_result.distances.clear(); - topk_query_result.ids.clear(); - topk_query_result.row_num = 0; + topk_query_result.clear(); std::vector query_range_array; milvus::Range rg; diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 4a9c319b4d..fd19281343 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -250,12 +250,17 @@ ClientProxy::Search(const std::string& table_name, const std::vectorSearch(result, search_param); // step 4: convert result array - topk_query_result.row_num = result.row_num(); - topk_query_result.ids.resize(result.ids().size()); - memcpy(topk_query_result.ids.data(), result.ids().data(), result.ids().size() * sizeof(int64_t)); - topk_query_result.distances.resize(result.distances().size()); - memcpy(topk_query_result.distances.data(), result.distances().data(), - result.distances().size() * sizeof(float)); + topk_query_result.reserve(result.row_num()); + int64_t nq = result.row_num(); + int64_t topk = result.ids().size() / nq; + for (int64_t i = 0; i < result.row_num(); i++) { + milvus::QueryResult one_result; + one_result.ids.resize(topk); + one_result.distances.resize(topk); + memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t)); + memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float)); + topk_query_result.emplace_back(one_result); + } return status; } catch (std::exception& ex) { diff --git a/core/src/sdk/include/MilvusApi.h b/core/src/sdk/include/MilvusApi.h index 0ec37fa9a4..9fa98deb40 100644 --- a/core/src/sdk/include/MilvusApi.h +++ b/core/src/sdk/include/MilvusApi.h @@ -81,11 +81,11 @@ struct RowRecord { /** * @brief TopK query result */ -struct TopKQueryResult { - int64_t row_num; +struct QueryResult { std::vector ids; std::vector distances; }; +using TopKQueryResult = std::vector; /** * @brief index parameters From 38da8dbb1df8effe48129050583e7fcb08a6806c Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Wed, 20 Nov 2019 18:37:06 +0800 Subject: [PATCH 67/82] fix --- core/src/wrapper/VecImpl.cpp | 125 ----------------------------------- 1 file changed, 125 deletions(-) diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index 6d681f7600..74e9e94a2f 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -248,130 +248,5 @@ BFIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const return Status::OK(); } -// TODO(linxj): add lock here. -Status -IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, - const float* xt) { - try { - dim = cfg->d; - auto dataset = GenDatasetWithIds(nb, dim, xb, ids); - - auto preprocessor = index_->BuildPreprocessor(dataset, cfg); - index_->set_preprocessor(preprocessor); - auto model = index_->Train(dataset, cfg); - index_->set_index_model(model); - index_->Add(dataset, cfg); - - if (auto device_index = std::dynamic_pointer_cast(index_)) { - auto host_index = device_index->CopyGpuToCpu(Config()); - index_ = host_index; - type = ConvertToCpuIndexType(type); - } else { - WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed"; - return Status(KNOWHERE_ERROR, "Build IVFMIXIndex Failed"); - } - } catch (knowhere::KnowhereException& e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (std::exception& e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_ERROR, e.what()); - } - return Status::OK(); -} - -Status -IVFMixIndex::Load(const knowhere::BinarySet& index_binary) { - index_->Load(index_binary); - dim = Dimension(); - return Status::OK(); -} - -knowhere::QuantizerPtr -IVFHybridIndex::LoadQuantizer(const Config& conf) { - // TODO(linxj): Hardcode here - if (auto new_idx = std::dynamic_pointer_cast(index_)) { - return new_idx->LoadQuantizer(conf); - } else { - WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); - } -} - -Status -IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) { - try { - // TODO(linxj): Hardcode here - if (auto new_idx = std::dynamic_pointer_cast(index_)) { - new_idx->SetQuantizer(q); - } else { - WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); - return Status(KNOWHERE_ERROR, "not supported"); - } - } catch (knowhere::KnowhereException& e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (std::exception& e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_ERROR, e.what()); - } - return Status::OK(); -} - -Status -IVFHybridIndex::UnsetQuantizer() { - try { - // TODO(linxj): Hardcode here - if (auto new_idx = std::dynamic_pointer_cast(index_)) { - new_idx->UnsetQuantizer(); - } else { - WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); - return Status(KNOWHERE_ERROR, "not supported"); - } - } catch (knowhere::KnowhereException& e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (std::exception& e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_ERROR, e.what()); - } - return Status::OK(); -} - -VecIndexPtr -IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { - try { - // TODO(linxj): Hardcode here - if (auto new_idx = std::dynamic_pointer_cast(index_)) { - return std::make_shared(new_idx->LoadData(q, conf), type); - } else { - WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); - } - } catch (knowhere::KnowhereException& e) { - WRAPPER_LOG_ERROR << e.what(); - } catch (std::exception& e) { - WRAPPER_LOG_ERROR << e.what(); - } - return nullptr; -} - -std::pair -IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& cfg) { - try { - // TODO(linxj): Hardcode here - if (auto hybrid_idx = std::dynamic_pointer_cast(index_)) { - auto pair = hybrid_idx->CopyCpuToGpuWithQuantizer(device_id, cfg); - auto new_idx = std::make_shared(pair.first, type); - return std::make_pair(new_idx, pair.second); - } else { - WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); - } - } catch (knowhere::KnowhereException& e) { - WRAPPER_LOG_ERROR << e.what(); - } catch (std::exception& e) { - WRAPPER_LOG_ERROR << e.what(); - } - return std::make_pair(nullptr, nullptr); -} - } // namespace engine } // namespace milvus From 8a42c8759e8ed22af676ba1857caef6073832615 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Wed, 20 Nov 2019 18:44:11 +0800 Subject: [PATCH 68/82] Query API in customization still uses old version --- CHANGELOG.md | 1 + .../scheduler/action/PushTaskToNeighbour.cpp | 35 ------------------- core/unittest/db/test_db.cpp | 18 ++++++---- 3 files changed, 13 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42d5b8978f..b6c1e323ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#353 - Rename config.h.in to version.h.in - \#374 - sdk_simple return empty result - \#397 - sdk_simple return incorrect result +- \#440 - Query API in customization still uses old version ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/scheduler/action/PushTaskToNeighbour.cpp b/core/src/scheduler/action/PushTaskToNeighbour.cpp index f49f1d871f..562041cee1 100644 --- a/core/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/core/src/scheduler/action/PushTaskToNeighbour.cpp @@ -106,41 +106,6 @@ Action::SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, Resou std::shared_ptr event) { auto task_item = event->task_table_item_; auto task = event->task_table_item_->task; - // if (resource->type() == ResourceType::DISK) { - // // step 1: calculate shortest path per resource, from disk to compute resource - // auto compute_resources = res_mgr->GetComputeResources(); - // std::vector> paths; - // std::vector transport_costs; - // for (auto& res : compute_resources) { - // std::vector path; - // uint64_t transport_cost = ShortestPath(resource, res, res_mgr, path); - // transport_costs.push_back(transport_cost); - // paths.emplace_back(path); - // } - // if (task->job_.lock()->type() == JobType::BUILD) { - // // step2: Read device id in config - // // get build index gpu resource - // server::Config& config = server::Config::GetInstance(); - // int32_t build_index_gpu; - // Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); - // - // bool find_gpu_res = false; - // if (res_mgr->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { - // for (uint64_t i = 0; i < compute_resources.size(); ++i) { - // if (compute_resources[i]->name() == - // res_mgr->GetResource(ResourceType::GPU, build_index_gpu)->name()) { - // find_gpu_res = true; - // Path task_path(paths[i], paths[i].size() - 1); - // task->path() = task_path; - // break; - // } - // } - // } - // if (not find_gpu_res) { - // task->path() = Path(paths[0], paths[0].size() - 1); - // } - // } - // } if (resource->name() == task->path().Last()) { resource->WakeupExecutor(); diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index d8614dd5d1..9e01b5ea1a 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -305,24 +305,30 @@ TEST_F(DBTest, SEARCH_TEST) { // test FAISS_IVFSQ8H optimizer index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H; db_->CreateIndex(TABLE_NAME, index); // wait until build index finish + std::vector partition_tag; + milvus::engine::ResultIds result_ids; + milvus::engine::ResultDistances result_dists; { - milvus::engine::QueryResults results; - stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results); + result_ids.clear(); + result_dists.clear(); + stat = db_->Query(TABLE_NAME, partition_tag, k, nq, 10, xq.data(), result_ids, result_dists); ASSERT_TRUE(stat.ok()); } { - milvus::engine::QueryResults large_nq_results; - stat = db_->Query(TABLE_NAME, k, 200, 10, xq.data(), large_nq_results); + result_ids.clear(); + result_dists.clear(); + stat = db_->Query(TABLE_NAME, partition_tag, k, 200, 10, xq.data(), result_ids, result_dists); ASSERT_TRUE(stat.ok()); } { // search by specify index file milvus::engine::meta::DatesT dates; std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; - milvus::engine::QueryResults results; - stat = db_->Query(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, results); + result_ids.clear(); + result_dists.clear(); + stat = db_->QueryByFileID(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, result_ids, result_dists); ASSERT_TRUE(stat.ok()); } From 3d10c48380caf9e6f06bfcebfe44ffbe29d38064 Mon Sep 17 00:00:00 2001 From: groot Date: Wed, 20 Nov 2019 19:07:11 +0800 Subject: [PATCH 69/82] Move migration folder into scripts folder --- core/{ => scripts}/migration/README.md | 0 core/{ => scripts}/migration/mysql_4_to_6.sql | 0 core/{ => scripts}/migration/sqlite_4_to_6.sql | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename core/{ => scripts}/migration/README.md (100%) rename core/{ => scripts}/migration/mysql_4_to_6.sql (100%) rename core/{ => scripts}/migration/sqlite_4_to_6.sql (100%) diff --git a/core/migration/README.md b/core/scripts/migration/README.md similarity index 100% rename from core/migration/README.md rename to core/scripts/migration/README.md diff --git a/core/migration/mysql_4_to_6.sql b/core/scripts/migration/mysql_4_to_6.sql similarity index 100% rename from core/migration/mysql_4_to_6.sql rename to core/scripts/migration/mysql_4_to_6.sql diff --git a/core/migration/sqlite_4_to_6.sql b/core/scripts/migration/sqlite_4_to_6.sql similarity index 100% rename from core/migration/sqlite_4_to_6.sql rename to core/scripts/migration/sqlite_4_to_6.sql From de6d32171f2577ead8c1f091b0065ea14d4030d6 Mon Sep 17 00:00:00 2001 From: groot Date: Wed, 20 Nov 2019 19:54:18 +0800 Subject: [PATCH 70/82] add script for meta migration --- core/scripts/migration/mysql_6_to_4.sql | 3 +++ core/scripts/migration/sqlite_6_to_4.sql | 7 +++++++ 2 files changed, 10 insertions(+) create mode 100644 core/scripts/migration/mysql_6_to_4.sql create mode 100644 core/scripts/migration/sqlite_6_to_4.sql diff --git a/core/scripts/migration/mysql_6_to_4.sql b/core/scripts/migration/mysql_6_to_4.sql new file mode 100644 index 0000000000..96c60e0280 --- /dev/null +++ b/core/scripts/migration/mysql_6_to_4.sql @@ -0,0 +1,3 @@ +alter table Tables drop column owner_table; +alter table Tables drop column partition_tag; +alter table Tables drop column version; diff --git a/core/scripts/migration/sqlite_6_to_4.sql b/core/scripts/migration/sqlite_6_to_4.sql new file mode 100644 index 0000000000..686d276f46 --- /dev/null +++ b/core/scripts/migration/sqlite_6_to_4.sql @@ -0,0 +1,7 @@ +CREATE TABLE 'TempTables' ( 'id' INTEGER PRIMARY KEY NOT NULL , 'table_id' TEXT UNIQUE NOT NULL , 'state' INTEGER NOT NULL , 'dimension' INTEGER NOT NULL , 'created_on' INTEGER NOT NULL , 'flag' INTEGER DEFAULT 0 NOT NULL , 'index_file_size' INTEGER NOT NULL , 'engine_type' INTEGER NOT NULL , 'nlist' INTEGER NOT NULL , 'metric_type' INTEGER NOT NULL); + +INSERT INTO TempTables SELECT id, table_id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type FROM Tables; + +DROP TABLE Tables; + +ALTER TABLE TempTables RENAME TO Tables; From 047f4bb43ee2cd7eab07f229ada1e6f167d608bb Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 21 Nov 2019 10:25:38 +0800 Subject: [PATCH 71/82] mark off faiss version --- ci/jenkins/Jenkinsfile | 40 +++++++++---------- .../index/cmake/ThirdPartyPackagesCore.cmake | 11 ++++- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index bf7a70d11a..8d3953b112 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -50,7 +50,7 @@ pipeline { } stages { - stage("Run GPU Version Build") { + stage("Run Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -60,7 +60,7 @@ pipeline { } stages { - stage('GPU Version Build') { + stage('Build') { steps { container('milvus-build-env') { script { @@ -69,7 +69,7 @@ pipeline { } } } - stage('GPU Version Code Coverage') { + stage('Code Coverage') { steps { container('milvus-build-env') { script { @@ -78,7 +78,7 @@ pipeline { } } } - stage('Upload GPU Version Package') { + stage('Upload Package') { steps { container('milvus-build-env') { script { @@ -90,7 +90,7 @@ pipeline { } } - stage("Publish GPU Version docker images") { + stage("Publish docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -100,7 +100,7 @@ pipeline { } stages { - stage('Publish GPU Version') { + stage('Publish') { steps { container('publish-images'){ script { @@ -112,7 +112,7 @@ pipeline { } } - stage("Deploy GPU Version to Development") { + stage("Deploy to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -122,7 +122,7 @@ pipeline { } stages { - stage("Deploy GPU Version to Dev") { + stage("Deploy to Dev") { steps { container('milvus-test-env') { script { @@ -132,7 +132,7 @@ pipeline { } } - stage("GPU Version Dev Test") { + stage("Dev Test") { steps { container('milvus-test-env') { script { @@ -147,7 +147,7 @@ pipeline { } } - stage ("Cleanup GPU Version Dev") { + stage ("Cleanup Dev") { steps { container('milvus-test-env') { script { @@ -180,7 +180,7 @@ pipeline { } stages { - stage("Run CPU Version Build") { + stage("Run Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -190,7 +190,7 @@ pipeline { } stages { - stage('Build CPU Version') { + stage('Build') { steps { container('milvus-build-env') { script { @@ -199,7 +199,7 @@ pipeline { } } } - stage('CPU Version Code Coverage') { + stage('Code Coverage') { steps { container('milvus-build-env') { script { @@ -208,7 +208,7 @@ pipeline { } } } - stage('Upload CPU Version Package') { + stage('Upload Package') { steps { container('milvus-build-env') { script { @@ -220,7 +220,7 @@ pipeline { } } - stage("Publish CPU Version docker images") { + stage("Publish docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -230,7 +230,7 @@ pipeline { } stages { - stage('Publish CPU Version') { + stage('Publish') { steps { container('publish-images'){ script { @@ -242,7 +242,7 @@ pipeline { } } - stage("Deploy CPU Version to Development") { + stage("Deploy to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -252,7 +252,7 @@ pipeline { } stages { - stage("Deploy CPU Version to Dev") { + stage("Deploy to Dev") { steps { container('milvus-test-env') { script { @@ -262,7 +262,7 @@ pipeline { } } - stage("CPU Version Dev Test") { + stage("Dev Test") { steps { container('milvus-test-env') { script { @@ -277,7 +277,7 @@ pipeline { } } - stage ("Cleanup CPU Version Dev") { + stage ("Cleanup Dev") { steps { container('milvus-test-env') { script { diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index c046bc3a56..1a22a9d2be 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -733,7 +733,16 @@ macro(build_faiss) if (USE_JFROG_CACHE STREQUAL "ON") string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}") - set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz") + if (KNOWHERE_GPU_VERSION) + set(FAISS_COMPUTE_TYPE "gpu") + else () + set(FAISS_COMPUTE_TYPE "cpu") + endif() + if (FAISS_WITH_MKL) + set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMPUTE_TYPE}_mkl_${FAISS_COMBINE_MD5}.tar.gz") + else () + set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMPUTE_TYPE}_openblas_${FAISS_COMBINE_MD5}.tar.gz") + endif() set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}") set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}") From fd35ee6de8b3544bdedf3d543f050d6c58ad9117 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 21 Nov 2019 10:49:52 +0800 Subject: [PATCH 72/82] update ci/scripts/coverage.sh --- ci/scripts/coverage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/coverage.sh b/ci/scripts/coverage.sh index dcd460eaeb..cece6f0dcc 100755 --- a/ci/scripts/coverage.sh +++ b/ci/scripts/coverage.sh @@ -116,7 +116,7 @@ for test in `ls ${DIR_UNITTEST}`; do if [ $? -ne 0 ]; then echo ${args} echo ${DIR_UNITTEST}/${test} "run failed" - exit -1 + exit 1 fi done @@ -143,7 +143,7 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ if [ $? -ne 0 ]; then echo "gen ${FILE_INFO_OUTPUT_NEW} failed" - exit -2 + exit 2 fi # gen html report From 7fa7a2a65bd697585a62f93ed0af40cc3a18ceff Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 21 Nov 2019 11:28:28 +0800 Subject: [PATCH 73/82] #449 Add ShowPartitions example for C++ SDK --- CHANGELOG.md | 1 + core/src/sdk/examples/partition/src/ClientTest.cpp | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8b243546e..e7c52eb5bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#404 - Add virtual method Init() in Pass abstract class - \#409 - Add a Fallback pass in optimizer - \#433 - C++ SDK query result is not easy to use +- \#449 - Add ShowPartitions example for C++ SDK ## Task diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index a12a7ff50e..ecbf0a80e7 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -93,6 +93,15 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "CreatePartition function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintPartitionParam(partition_param); } + + // show partitions + milvus::PartitionList partition_array; + stat = conn->ShowPartitions(TABLE_NAME, partition_array); + + std::cout << partition_array.size() << " partitions created:" << std::endl; + for (auto& partition : partition_array) { + std::cout << "\t" << partition.partition_name << "\t tag = " << partition.partition_tag << std::endl; + } } { // insert vectors From 0c1851d1be18a5a7aeeeb008bb07372303a03280 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 21 Nov 2019 11:58:51 +0800 Subject: [PATCH 74/82] format Jenkins CI --- ci/jenkins/step/singleDevNightlyTest.groovy | 2 +- ci/jenkins/step/singleDevTest.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index e27bc6c0a5..9d1ca3546a 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -4,7 +4,7 @@ timeout(time: 90, unit: 'MINUTES') { sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-milvus-engine.milvus.svc.cluster.local" } // mysql database backend test - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" + load "ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" if (!fileExists('milvus-helm')) { dir ("milvus-helm") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 08812ae01f..ea1b756de1 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -5,7 +5,7 @@ timeout(time: 60, unit: 'MINUTES') { } // mysql database backend test - // load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" + // load "ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" // if (!fileExists('milvus-helm')) { // dir ("milvus-helm") { From 046c5543b8e5a6c79a3cad7b670f89e16c7ae69c Mon Sep 17 00:00:00 2001 From: "peng.xu" Date: Thu, 21 Nov 2019 13:56:37 +0800 Subject: [PATCH 75/82] [skip ci] (doc/shards): update doc and demo --- shards/README.md | 2 +- shards/README_CN.md | 2 +- shards/all_in_one/all_in_one.yml | 5 ++-- shards/all_in_one/ro_server.yml | 8 +++---- shards/all_in_one/wr_server.yml | 41 ++++++++++++++++++++++++++++++++ shards/mishards/.env.example | 4 ++-- 6 files changed, 52 insertions(+), 10 deletions(-) create mode 100644 shards/all_in_one/wr_server.yml diff --git a/shards/README.md b/shards/README.md index f59eca0460..1e0b000af5 100644 --- a/shards/README.md +++ b/shards/README.md @@ -54,7 +54,7 @@ Follow below steps to start a standalone Milvus instance with Mishards from sour 3. Start Milvus server. ```shell - $ sudo nvidia-docker run --rm -d -p 19530:19530 -v /tmp/milvus/db:/opt/milvus/db milvusdb/milvus:0.5.0-d102119-ede20b + $ sudo nvidia-docker run --rm -d -p 19530:19530 -v /tmp/milvus/db:/opt/milvus/db milvusdb/milvus ``` 4. Update path permissions. diff --git a/shards/README_CN.md b/shards/README_CN.md index 24e019d001..98264b206b 100644 --- a/shards/README_CN.md +++ b/shards/README_CN.md @@ -48,7 +48,7 @@ Python 版本为3.6及以上。 3. 启动 Milvus 服务。 ```shell - $ sudo nvidia-docker run --rm -d -p 19530:19530 -v /tmp/milvus/db:/opt/milvus/db milvusdb/milvus:0.5.0-d102119-ede20b + $ sudo nvidia-docker run --rm -d -p 19530:19530 -v /tmp/milvus/db:/opt/milvus/db milvusdb/milvus ``` 4. 更改目录权限。 diff --git a/shards/all_in_one/all_in_one.yml b/shards/all_in_one/all_in_one.yml index 40473fe8b9..75a3340068 100644 --- a/shards/all_in_one/all_in_one.yml +++ b/shards/all_in_one/all_in_one.yml @@ -3,14 +3,15 @@ services: milvus_wr: runtime: nvidia restart: always - image: milvusdb/milvus:0.5.0-d102119-ede20b + image: milvusdb/milvus volumes: - /tmp/milvus/db:/opt/milvus/db + - ./wr_server.yml:/opt/milvus/conf/server_config.yaml milvus_ro: runtime: nvidia restart: always - image: milvusdb/milvus:0.5.0-d102119-ede20b + image: milvusdb/milvus volumes: - /tmp/milvus/db:/opt/milvus/db - ./ro_server.yml:/opt/milvus/conf/server_config.yaml diff --git a/shards/all_in_one/ro_server.yml b/shards/all_in_one/ro_server.yml index 10cf695448..09857ee9c8 100644 --- a/shards/all_in_one/ro_server.yml +++ b/shards/all_in_one/ro_server.yml @@ -12,7 +12,7 @@ db_config: # Keep 'dialect://:@:/', and replace other texts with real values # Replace 'dialect' with 'mysql' or 'sqlite' - insert_buffer_size: 4 # GB, maximum insert buffer size allowed + insert_buffer_size: 1 # GB, maximum insert buffer size allowed # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory preload_table: # preload data at startup, '*' means load all tables, empty value means no preload @@ -25,14 +25,14 @@ metric_config: port: 8080 # port prometheus uses to fetch metrics cache_config: - cpu_cache_capacity: 16 # GB, CPU memory used for cache + cpu_cache_capacity: 4 # GB, CPU memory used for cache cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - gpu_cache_capacity: 4 # GB, GPU memory used for cache + gpu_cache_capacity: 1 # GB, GPU memory used for cache gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered cache_insert_data: false # whether to load inserted data into cache engine_config: - use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times + use_blas_threshold: 800 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times resource_config: diff --git a/shards/all_in_one/wr_server.yml b/shards/all_in_one/wr_server.yml new file mode 100644 index 0000000000..5d7d855c03 --- /dev/null +++ b/shards/all_in_one/wr_server.yml @@ -0,0 +1,41 @@ +server_config: + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # port range: 1025 ~ 65534 + deploy_mode: cluster_writable # deployment type: single, cluster_readonly, cluster_writable + time_zone: UTC+8 + +db_config: + primary_path: /opt/milvus # path used to store data and meta + secondary_path: # path used to store data only, split by semicolon + + backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database + # Keep 'dialect://:@:/', and replace other texts with real values + # Replace 'dialect' with 'mysql' or 'sqlite' + + insert_buffer_size: 2 # GB, maximum insert buffer size allowed + # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory + + preload_table: # preload data at startup, '*' means load all tables, empty value means no preload + # you can specify preload tables like this: table1,table2,table3 + +metric_config: + enable_monitor: false # enable monitoring or not + collector: prometheus # prometheus + prometheus_config: + port: 8080 # port prometheus uses to fetch metrics + +cache_config: + cpu_cache_capacity: 2 # GB, CPU memory used for cache + cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + gpu_cache_capacity: 2 # GB, GPU memory used for cache + gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + cache_insert_data: false # whether to load inserted data into cache + +engine_config: + use_blas_threshold: 800 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times + # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times + +resource_config: + search_resources: # define the GPUs used for search computation, valid value: gpux + - gpu0 + index_build_device: gpu0 # GPU used for building index diff --git a/shards/mishards/.env.example b/shards/mishards/.env.example index f1c812a269..91b67760af 100644 --- a/shards/mishards/.env.example +++ b/shards/mishards/.env.example @@ -1,7 +1,7 @@ DEBUG=True WOSERVER=tcp://127.0.0.1:19530 -SERVER_PORT=19532 +SERVER_PORT=19535 SERVER_TEST_PORT=19888 #SQLALCHEMY_DATABASE_URI=mysql+pymysql://root:root@127.0.0.1:3306/milvus?charset=utf8mb4 @@ -19,7 +19,7 @@ TRACER_CLASS_NAME=jaeger TRACING_SERVICE_NAME=fortest TRACING_SAMPLER_TYPE=const TRACING_SAMPLER_PARAM=1 -TRACING_LOG_PAYLOAD=True +TRACING_LOG_PAYLOAD=False #TRACING_SAMPLER_TYPE=probabilistic #TRACING_SAMPLER_PARAM=0.5 From 1c6b986e37137176d54ae4c25b540cb787f687f9 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 21 Nov 2019 14:49:23 +0800 Subject: [PATCH 76/82] Fix Domain host set not correct in CI bug --- ci/jenkins/step/singleDevNightlyTest.groovy | 4 ++-- ci/jenkins/step/singleDevTest.groovy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index 9d1ca3546a..cee8a092c1 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -1,7 +1,7 @@ timeout(time: 90, unit: 'MINUTES') { dir ("tests/milvus_python_test") { sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-milvus-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" } // mysql database backend test load "ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" @@ -17,6 +17,6 @@ timeout(time: 90, unit: 'MINUTES') { } } dir ("tests/milvus_python_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-milvus-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" } } diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index ea1b756de1..db0fdc0f3b 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -1,7 +1,7 @@ timeout(time: 60, unit: 'MINUTES') { dir ("tests/milvus_python_test") { sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-milvus-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" } // mysql database backend test @@ -18,6 +18,6 @@ timeout(time: 60, unit: 'MINUTES') { // } // } // dir ("tests/milvus_python_test") { - // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-milvus-engine.milvus.svc.cluster.local" + // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" // } } From 1ac30913e73d9eb621eb770a7dd0125fd5c2c6a8 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Thu, 21 Nov 2019 15:06:00 +0800 Subject: [PATCH 77/82] move seal to Load --- CHANGELOG.md | 1 + .../knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp | 4 +++- .../knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp | 3 --- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8b243546e..af4abe71a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#412 - Message returned is confused when partition created with null partition name - \#416 - Drop the same partition success repeatally - \#440 - Query API in customization still uses old version +- \#458 - Index data is not compatible between 0.5 and 0.6 ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp b/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp index 783487be3a..8fce37a81e 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp @@ -33,7 +33,7 @@ FaissBaseIndex::SerializeImpl() { try { faiss::Index* index = index_.get(); - SealImpl(); + // SealImpl(); MemoryIOWriter writer; faiss::write_index(index, &writer); @@ -60,6 +60,8 @@ FaissBaseIndex::LoadImpl(const BinarySet& index_binary) { faiss::Index* index = faiss::read_index(&reader); index_.reset(index); + + SealImpl(); } void diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index 251dfc12ed..d69f87a061 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -86,9 +86,6 @@ GPUIVF::SerializeImpl() { faiss::Index* index = index_.get(); faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(index); - // TODO(linxj): support seal - // SealImpl(); - faiss::write_index(host_index, &writer); delete host_index; } From 098ba111d7437f16e2614cd28b0610cfcdf7b608 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 21 Nov 2019 15:42:10 +0800 Subject: [PATCH 78/82] format Jenkinsfile --- ci/jenkins/Jenkinsfile | 24 +++--- .../pod/milvus-cpu-version-build-env-pod.yaml | 2 +- .../pod/milvus-gpu-version-build-env-pod.yaml | 2 +- ci/jenkins/step/publishImages.groovy | 78 +++++++++---------- 4 files changed, 52 insertions(+), 54 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 8d3953b112..47be9f5bb6 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -53,7 +53,7 @@ pipeline { stage("Run Build") { agent { kubernetes { - label "${BINRARY_VERSION}-build" + label "${env.BINRARY_VERSION}-build" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml' } @@ -62,7 +62,7 @@ pipeline { stages { stage('Build') { steps { - container('milvus-build-env') { + container("milvus-${env.BINRARY_VERSION}-build-env") { script { load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" } @@ -71,7 +71,7 @@ pipeline { } stage('Code Coverage') { steps { - container('milvus-build-env') { + container("milvus-${env.BINRARY_VERSION}-build-env") { script { load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" } @@ -80,7 +80,7 @@ pipeline { } stage('Upload Package') { steps { - container('milvus-build-env') { + container("milvus-${env.BINRARY_VERSION}-build-env") { script { load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" } @@ -93,7 +93,7 @@ pipeline { stage("Publish docker images") { agent { kubernetes { - label "${BINRARY_VERSION}-publish" + label "${env.BINRARY_VERSION}-publish" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/docker-pod.yaml' } @@ -115,7 +115,7 @@ pipeline { stage("Deploy to Development") { agent { kubernetes { - label "${BINRARY_VERSION}-dev-test" + label "${env.BINRARY_VERSION}-dev-test" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/testEnvironment.yaml' } @@ -183,7 +183,7 @@ pipeline { stage("Run Build") { agent { kubernetes { - label "${BINRARY_VERSION}-build" + label "${env.BINRARY_VERSION}-build" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml' } @@ -192,7 +192,7 @@ pipeline { stages { stage('Build') { steps { - container('milvus-build-env') { + container("milvus-${env.BINRARY_VERSION}-build-env") { script { load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" } @@ -201,7 +201,7 @@ pipeline { } stage('Code Coverage') { steps { - container('milvus-build-env') { + container("milvus-${env.BINRARY_VERSION}-build-env") { script { load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" } @@ -210,7 +210,7 @@ pipeline { } stage('Upload Package') { steps { - container('milvus-build-env') { + container("milvus-${env.BINRARY_VERSION}-build-env") { script { load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" } @@ -223,7 +223,7 @@ pipeline { stage("Publish docker images") { agent { kubernetes { - label "${BINRARY_VERSION}-publish" + label "${env.BINRARY_VERSION}-publish" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/docker-pod.yaml' } @@ -245,7 +245,7 @@ pipeline { stage("Deploy to Development") { agent { kubernetes { - label "${BINRARY_VERSION}-dev-test" + label "${env.BINRARY_VERSION}-dev-test" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/testEnvironment.yaml' } diff --git a/ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml index 561bfe8140..894067d66c 100644 --- a/ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml @@ -7,7 +7,7 @@ metadata: componet: cpu-build-env spec: containers: - - name: milvus-build-env + - name: milvus-cpu-build-env image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.6.0-ubuntu18.04 env: - name: POD_IP diff --git a/ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml index 422dd72ab2..f5ceb9462b 100644 --- a/ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml @@ -7,7 +7,7 @@ metadata: componet: gpu-build-env spec: containers: - - name: milvus-build-env + - name: milvus-gpu-build-env image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.6.0-ubuntu18.04 env: - name: POD_IP diff --git a/ci/jenkins/step/publishImages.groovy b/ci/jenkins/step/publishImages.groovy index 72e9924c62..5449bcedd8 100644 --- a/ci/jenkins/step/publishImages.groovy +++ b/ci/jenkins/step/publishImages.groovy @@ -1,47 +1,45 @@ -container('publish-images') { - timeout(time: 15, unit: 'MINUTES') { - dir ("docker/deploy/${env.BINRARY_VERSION}/${env.OS_NAME}") { - def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" +timeout(time: 15, unit: 'MINUTES') { + dir ("docker/deploy/${env.BINRARY_VERSION}/${env.OS_NAME}") { + def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" - withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { - def downloadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage}") + withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { + def downloadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage}") - if (downloadStatus != 0) { - error("\" Download \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage} \" failed!") - } + if (downloadStatus != 0) { + error("\" Download \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage} \" failed!") } - sh "tar zxvf ${binaryPackage}" - def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}" + } + sh "tar zxvf ${binaryPackage}" + def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}" - try { - def isExistSourceImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName} 2>&1 > /dev/null") - if (isExistSourceImage == 0) { - def removeSourceImageStatus = sh(returnStatus: true, script: "docker rmi ${imageName}") - } - - def customImage = docker.build("${imageName}") - - def isExistTargeImage = sh(returnStatus: true, script: "docker inspect --type=image ${params.DOKCER_REGISTRY_URL}/${imageName} 2>&1 > /dev/null") - if (isExistTargeImage == 0) { - def removeTargeImageStatus = sh(returnStatus: true, script: "docker rmi ${params.DOKCER_REGISTRY_URL}/${imageName}") - } - - docker.withRegistry("https://${params.DOKCER_REGISTRY_URL}", "${params.DOCKER_CREDENTIALS_ID}") { - customImage.push() - } - } catch (exc) { - throw exc - } finally { - def isExistSourceImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName} 2>&1 > /dev/null") - if (isExistSourceImage == 0) { - def removeSourceImageStatus = sh(returnStatus: true, script: "docker rmi ${imageName}") - } - - def isExistTargeImage = sh(returnStatus: true, script: "docker inspect --type=image ${params.DOKCER_REGISTRY_URL}/${imageName} 2>&1 > /dev/null") - if (isExistTargeImage == 0) { - def removeTargeImageStatus = sh(returnStatus: true, script: "docker rmi ${params.DOKCER_REGISTRY_URL}/${imageName}") - } + try { + def isExistSourceImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName} 2>&1 > /dev/null") + if (isExistSourceImage == 0) { + def removeSourceImageStatus = sh(returnStatus: true, script: "docker rmi ${imageName}") } - } + + def customImage = docker.build("${imageName}") + + def isExistTargeImage = sh(returnStatus: true, script: "docker inspect --type=image ${params.DOKCER_REGISTRY_URL}/${imageName} 2>&1 > /dev/null") + if (isExistTargeImage == 0) { + def removeTargeImageStatus = sh(returnStatus: true, script: "docker rmi ${params.DOKCER_REGISTRY_URL}/${imageName}") + } + + docker.withRegistry("https://${params.DOKCER_REGISTRY_URL}", "${params.DOCKER_CREDENTIALS_ID}") { + customImage.push() + } + } catch (exc) { + throw exc + } finally { + def isExistSourceImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName} 2>&1 > /dev/null") + if (isExistSourceImage == 0) { + def removeSourceImageStatus = sh(returnStatus: true, script: "docker rmi ${imageName}") + } + + def isExistTargeImage = sh(returnStatus: true, script: "docker inspect --type=image ${params.DOKCER_REGISTRY_URL}/${imageName} 2>&1 > /dev/null") + if (isExistTargeImage == 0) { + def removeTargeImageStatus = sh(returnStatus: true, script: "docker rmi ${params.DOKCER_REGISTRY_URL}/${imageName}") + } + } } } From 3285aa98851beb7f3fbb8ea5d862ba1b30ece3ee Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 21 Nov 2019 16:08:14 +0800 Subject: [PATCH 79/82] format Jenkinsfile --- ci/jenkins/step/cleanupSingleDev.groovy | 8 ++++---- ci/jenkins/step/deploySingle2Dev.groovy | 2 +- ci/jenkins/step/singleDevNightlyTest.groovy | 6 +++--- ci/jenkins/step/singleDevTest.groovy | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ci/jenkins/step/cleanupSingleDev.groovy b/ci/jenkins/step/cleanupSingleDev.groovy index 30325e0c91..3311592373 100644 --- a/ci/jenkins/step/cleanupSingleDev.groovy +++ b/ci/jenkins/step/cleanupSingleDev.groovy @@ -1,12 +1,12 @@ try { - def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}", returnStatus: true + def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}", returnStatus: true if (!helmResult) { - sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}" + sh "helm del --purge ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}" } } catch (exc) { - def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}", returnStatus: true + def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}", returnStatus: true if (!helmResult) { - sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}" + sh "helm del --purge ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}" } throw exc } diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index 7b479ff44a..929548d645 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,7 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index cee8a092c1..9944a8bac0 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -1,7 +1,7 @@ timeout(time: 90, unit: 'MINUTES') { dir ("tests/milvus_python_test") { sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" } // mysql database backend test load "ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" @@ -13,10 +13,10 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" } } diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index db0fdc0f3b..70223219a5 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -1,7 +1,7 @@ timeout(time: 60, unit: 'MINUTES') { dir ("tests/milvus_python_test") { sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" } // mysql database backend test @@ -14,10 +14,10 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { - // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" + // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" // } } From ae59ad1901a0dd98a33ba4ea72caea4e1bc7f1fe Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 21 Nov 2019 16:49:01 +0800 Subject: [PATCH 80/82] format Jenkinsfile --- ci/jenkins/Jenkinsfile | 8 ++++++++ ci/jenkins/step/cleanupSingleDev.groovy | 8 ++++---- ci/jenkins/step/deploySingle2Dev.groovy | 2 +- ci/jenkins/step/singleDevNightlyTest.groovy | 6 +++--- ci/jenkins/step/singleDevTest.groovy | 6 +++--- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 47be9f5bb6..01048bd953 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -113,6 +113,10 @@ pipeline { } stage("Deploy to Development") { + environment { + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() + } + agent { kubernetes { label "${env.BINRARY_VERSION}-dev-test" @@ -243,6 +247,10 @@ pipeline { } stage("Deploy to Development") { + environment { + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() + } + agent { kubernetes { label "${env.BINRARY_VERSION}-dev-test" diff --git a/ci/jenkins/step/cleanupSingleDev.groovy b/ci/jenkins/step/cleanupSingleDev.groovy index 3311592373..101105c027 100644 --- a/ci/jenkins/step/cleanupSingleDev.groovy +++ b/ci/jenkins/step/cleanupSingleDev.groovy @@ -1,12 +1,12 @@ try { - def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}", returnStatus: true + def helmResult = sh script: "helm status ${env.HELM_RELEASE_NAME}", returnStatus: true if (!helmResult) { - sh "helm del --purge ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}" + sh "helm del --purge ${env.HELM_RELEASE_NAME}" } } catch (exc) { - def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}", returnStatus: true + def helmResult = sh script: "helm status ${env.HELM_RELEASE_NAME}", returnStatus: true if (!helmResult) { - sh "helm del --purge ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}" + sh "helm del --purge ${env.HELM_RELEASE_NAME}" } throw exc } diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index 929548d645..cb2ad2b1cb 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,7 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index 9944a8bac0..d14ba1b66c 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -1,7 +1,7 @@ timeout(time: 90, unit: 'MINUTES') { dir ("tests/milvus_python_test") { sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local" } // mysql database backend test load "ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" @@ -13,10 +13,10 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local" } } diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 70223219a5..7b72eaacde 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -1,7 +1,7 @@ timeout(time: 60, unit: 'MINUTES') { dir ("tests/milvus_python_test") { sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local" } // mysql database backend test @@ -14,10 +14,10 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { - // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" + // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local" // } } From 32e5bba61aafcc1285acadfca63a28e39b9045a9 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Thu, 21 Nov 2019 16:54:51 +0800 Subject: [PATCH 81/82] fix --- core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 7f30a97ea0..8b734abdc6 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -97,7 +97,6 @@ IVF::Serialize() { } std::lock_guard lk(mutex_); - Seal(); return SerializeImpl(); } From acf4d0459d37a8760a4c931abeb29b7a4e9e916f Mon Sep 17 00:00:00 2001 From: zhenwu Date: Thu, 21 Nov 2019 17:23:06 +0800 Subject: [PATCH 82/82] Add partition case --- tests/milvus_python_test/pytest.ini | 2 +- tests/milvus_python_test/requirements.txt | 2 +- .../requirements_no_pymilvus.txt | 1 - tests/milvus_python_test/test_add_vectors.py | 96 +++- tests/milvus_python_test/test_connect.py | 13 +- tests/milvus_python_test/test_index.py | 216 ++++++++- tests/milvus_python_test/test_mix.py | 3 +- tests/milvus_python_test/test_partition.py | 431 ++++++++++++++++++ .../milvus_python_test/test_search_vectors.py | 238 +++++++++- tests/milvus_python_test/test_table.py | 2 +- tests/milvus_python_test/test_table_count.py | 88 +++- 11 files changed, 1065 insertions(+), 27 deletions(-) create mode 100644 tests/milvus_python_test/test_partition.py diff --git a/tests/milvus_python_test/pytest.ini b/tests/milvus_python_test/pytest.ini index 3f95dc29b8..3ae6a790db 100644 --- a/tests/milvus_python_test/pytest.ini +++ b/tests/milvus_python_test/pytest.ini @@ -4,6 +4,6 @@ log_format = [%(asctime)s-%(levelname)s-%(name)s]: %(message)s (%(filename)s:%(l log_cli = true log_level = 20 -timeout = 300 +timeout = 600 level = 1 \ No newline at end of file diff --git a/tests/milvus_python_test/requirements.txt b/tests/milvus_python_test/requirements.txt index c8fc02c096..016c8dedfc 100644 --- a/tests/milvus_python_test/requirements.txt +++ b/tests/milvus_python_test/requirements.txt @@ -22,4 +22,4 @@ wcwidth==0.1.7 wrapt==1.11.1 zipp==0.5.1 scikit-learn>=0.19.1 -pymilvus-test>=0.2.0 \ No newline at end of file +pymilvus-test>=0.2.0 diff --git a/tests/milvus_python_test/requirements_no_pymilvus.txt b/tests/milvus_python_test/requirements_no_pymilvus.txt index 45884c0c71..c6a933736e 100644 --- a/tests/milvus_python_test/requirements_no_pymilvus.txt +++ b/tests/milvus_python_test/requirements_no_pymilvus.txt @@ -17,7 +17,6 @@ allure-pytest==2.7.0 pytest-print==0.1.2 pytest-level==0.1.1 six==1.12.0 -thrift==0.11.0 typed-ast==1.3.5 wcwidth==0.1.7 wrapt==1.11.1 diff --git a/tests/milvus_python_test/test_add_vectors.py b/tests/milvus_python_test/test_add_vectors.py index f9f7f7d4ca..7245d51ea2 100644 --- a/tests/milvus_python_test/test_add_vectors.py +++ b/tests/milvus_python_test/test_add_vectors.py @@ -15,7 +15,7 @@ table_id = "test_add" ADD_TIMEOUT = 60 nprobe = 1 epsilon = 0.0001 - +tag = "1970-01-01" class TestAddBase: """ @@ -186,6 +186,7 @@ class TestAddBase: expected: status ok ''' index_param = get_simple_index_params + logging.getLogger().info(index_param) vector = gen_single_vector(dim) status, ids = connect.add_vectors(table, vector) status = connect.create_index(table, index_param) @@ -439,6 +440,80 @@ class TestAddBase: assert status.OK() assert len(ids) == nq + @pytest.mark.timeout(ADD_TIMEOUT) + def test_add_vectors_tag(self, connect, table): + ''' + target: test add vectors in table created before + method: create table and add vectors in it, with the partition_tag param + expected: the table row count equals to nq + ''' + nq = 5 + partition_name = gen_unique_str() + vectors = gen_vectors(nq, dim) + status = connect.create_partition(table, partition_name, tag) + status, ids = connect.add_vectors(table, vectors, partition_tag=tag) + assert status.OK() + assert len(ids) == nq + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_add_vectors_tag_A(self, connect, table): + ''' + target: test add vectors in table created before + method: create partition and add vectors in it + expected: the table row count equals to nq + ''' + nq = 5 + partition_name = gen_unique_str() + vectors = gen_vectors(nq, dim) + status = connect.create_partition(table, partition_name, tag) + status, ids = connect.add_vectors(partition_name, vectors) + assert status.OK() + assert len(ids) == nq + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_add_vectors_tag_not_existed(self, connect, table): + ''' + target: test add vectors in table created before + method: create table and add vectors in it, with the not existed partition_tag param + expected: status not ok + ''' + nq = 5 + vectors = gen_vectors(nq, dim) + status, ids = connect.add_vectors(table, vectors, partition_tag=tag) + assert not status.OK() + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_add_vectors_tag_not_existed_A(self, connect, table): + ''' + target: test add vectors in table created before + method: create partition, add vectors with the not existed partition_tag param + expected: status not ok + ''' + nq = 5 + vectors = gen_vectors(nq, dim) + new_tag = "new_tag" + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + status, ids = connect.add_vectors(table, vectors, partition_tag=new_tag) + assert not status.OK() + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_add_vectors_tag_existed(self, connect, table): + ''' + target: test add vectors in table created before + method: create table and add vectors in it repeatly, with the partition_tag param + expected: the table row count equals to nq + ''' + nq = 5 + partition_name = gen_unique_str() + vectors = gen_vectors(nq, dim) + status = connect.create_partition(table, partition_name, tag) + status, ids = connect.add_vectors(table, vectors, partition_tag=tag) + for i in range(5): + status, ids = connect.add_vectors(table, vectors, partition_tag=tag) + assert status.OK() + assert len(ids) == nq + @pytest.mark.level(2) def test_add_vectors_without_connect(self, dis_connect, table): ''' @@ -1198,7 +1273,8 @@ class TestAddAdvance: assert len(ids) == nb assert status.OK() -class TestAddTableNameInvalid(object): + +class TestNameInvalid(object): """ Test adding vectors with invalid table names """ @@ -1209,13 +1285,27 @@ class TestAddTableNameInvalid(object): def get_table_name(self, request): yield request.param + @pytest.fixture( + scope="function", + params=gen_invalid_table_names() + ) + def get_tag_name(self, request): + yield request.param + @pytest.mark.level(2) - def test_add_vectors_with_invalid_tablename(self, connect, get_table_name): + def test_add_vectors_with_invalid_table_name(self, connect, get_table_name): table_name = get_table_name vectors = gen_vectors(1, dim) status, result = connect.add_vectors(table_name, vectors) assert not status.OK() + @pytest.mark.level(2) + def test_add_vectors_with_invalid_tag_name(self, connect, get_tag_name): + tag_name = get_tag_name + vectors = gen_vectors(1, dim) + status, result = connect.add_vectors(table_name, vectors, partition_tag=tag_name) + assert not status.OK() + class TestAddTableVectorsInvalid(object): single_vector = gen_single_vector(dim) diff --git a/tests/milvus_python_test/test_connect.py b/tests/milvus_python_test/test_connect.py index dd7e80c1f9..143ac4d8bf 100644 --- a/tests/milvus_python_test/test_connect.py +++ b/tests/milvus_python_test/test_connect.py @@ -149,15 +149,14 @@ class TestConnect: milvus.connect(uri=uri_value, timeout=1) assert not milvus.connected() - # TODO: enable - def _test_connect_with_multiprocess(self, args): + def test_connect_with_multiprocess(self, args): ''' target: test uri connect with multiprocess method: set correct uri, test with multiprocessing connecting expected: all connection is connected ''' uri_value = "tcp://%s:%s" % (args["ip"], args["port"]) - process_num = 4 + process_num = 10 processes = [] def connect(milvus): @@ -248,7 +247,7 @@ class TestConnect: expected: connect raise an exception and connected is false ''' milvus = Milvus() - uri_value = "tcp://%s:19540" % args["ip"] + uri_value = "tcp://%s:39540" % args["ip"] with pytest.raises(Exception) as e: milvus.connect(host=args["ip"], port="", uri=uri_value) @@ -264,6 +263,7 @@ class TestConnect: milvus.connect(host="", port=args["port"], uri=uri_value, timeout=1) assert not milvus.connected() + # Disable, (issue: https://github.com/milvus-io/milvus/issues/288) def test_connect_param_priority_both_hostip_uri(self, args): ''' target: both host_ip_port / uri are both given, and not null, use the uri params @@ -273,8 +273,9 @@ class TestConnect: milvus = Milvus() uri_value = "tcp://%s:%s" % (args["ip"], args["port"]) with pytest.raises(Exception) as e: - milvus.connect(host=args["ip"], port=19540, uri=uri_value, timeout=1) - assert not milvus.connected() + res = milvus.connect(host=args["ip"], port=39540, uri=uri_value, timeout=1) + logging.getLogger().info(res) + # assert not milvus.connected() def _test_add_vector_and_disconnect_concurrently(self): ''' diff --git a/tests/milvus_python_test/test_index.py b/tests/milvus_python_test/test_index.py index 269e6137da..39aadb9d33 100644 --- a/tests/milvus_python_test/test_index.py +++ b/tests/milvus_python_test/test_index.py @@ -20,6 +20,7 @@ vectors = sklearn.preprocessing.normalize(vectors, axis=1, norm='l2') vectors = vectors.tolist() BUILD_TIMEOUT = 60 nprobe = 1 +tag = "1970-01-01" class TestIndexBase: @@ -62,6 +63,21 @@ class TestIndexBase: status = connect.create_index(table, index_params) assert status.OK() + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_partition(self, connect, table, get_index_params): + ''' + target: test create index interface + method: create table, create partition, and add vectors in it, create index + expected: return code equals to 0, and search success + ''' + partition_name = gen_unique_str() + index_params = get_index_params + logging.getLogger().info(index_params) + status = connect.create_partition(table, partition_name, tag) + status, ids = connect.add_vectors(table, vectors, partition_tag=tag) + status = connect.create_index(table, index_params) + assert status.OK() + @pytest.mark.level(2) def test_create_index_without_connect(self, dis_connect, table): ''' @@ -555,6 +571,21 @@ class TestIndexIP: status = connect.create_index(ip_table, index_params) assert status.OK() + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_partition(self, connect, ip_table, get_index_params): + ''' + target: test create index interface + method: create table, create partition, and add vectors in it, create index + expected: return code equals to 0, and search success + ''' + partition_name = gen_unique_str() + index_params = get_index_params + logging.getLogger().info(index_params) + status = connect.create_partition(ip_table, partition_name, tag) + status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) + status = connect.create_index(partition_name, index_params) + assert status.OK() + @pytest.mark.level(2) def test_create_index_without_connect(self, dis_connect, ip_table): ''' @@ -583,9 +614,9 @@ class TestIndexIP: query_vecs = [vectors[0], vectors[1], vectors[2]] top_k = 5 status, result = connect.search_vectors(ip_table, top_k, nprobe, query_vecs) + logging.getLogger().info(result) assert status.OK() assert len(result) == len(query_vecs) - # logging.getLogger().info(result) # TODO: enable @pytest.mark.timeout(BUILD_TIMEOUT) @@ -743,13 +774,13 @@ class TestIndexIP: ****************************************************************** """ - def test_describe_index(self, connect, ip_table, get_index_params): + def test_describe_index(self, connect, ip_table, get_simple_index_params): ''' target: test describe index interface method: create table and add vectors in it, create index, call describe index expected: return code 0, and index instructure ''' - index_params = get_index_params + index_params = get_simple_index_params logging.getLogger().info(index_params) status, ids = connect.add_vectors(ip_table, vectors) status = connect.create_index(ip_table, index_params) @@ -759,6 +790,80 @@ class TestIndexIP: assert result._table_name == ip_table assert result._index_type == index_params["index_type"] + def test_describe_index_partition(self, connect, ip_table, get_simple_index_params): + ''' + target: test describe index interface + method: create table, create partition and add vectors in it, create index, call describe index + expected: return code 0, and index instructure + ''' + partition_name = gen_unique_str() + index_params = get_simple_index_params + logging.getLogger().info(index_params) + status = connect.create_partition(ip_table, partition_name, tag) + status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) + status = connect.create_index(ip_table, index_params) + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == index_params["nlist"] + assert result._table_name == ip_table + assert result._index_type == index_params["index_type"] + status, result = connect.describe_index(partition_name) + logging.getLogger().info(result) + assert result._nlist == index_params["nlist"] + assert result._table_name == partition_name + assert result._index_type == index_params["index_type"] + + def test_describe_index_partition_A(self, connect, ip_table, get_simple_index_params): + ''' + target: test describe index interface + method: create table, create partition and add vectors in it, create index on partition, call describe index + expected: return code 0, and index instructure + ''' + partition_name = gen_unique_str() + index_params = get_simple_index_params + logging.getLogger().info(index_params) + status = connect.create_partition(ip_table, partition_name, tag) + status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) + status = connect.create_index(partition_name, index_params) + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == ip_table + assert result._index_type == IndexType.FLAT + status, result = connect.describe_index(partition_name) + logging.getLogger().info(result) + assert result._nlist == index_params["nlist"] + assert result._table_name == partition_name + assert result._index_type == index_params["index_type"] + + def test_describe_index_partition_B(self, connect, ip_table, get_simple_index_params): + ''' + target: test describe index interface + method: create table, create partitions and add vectors in it, create index on partitions, call describe index + expected: return code 0, and index instructure + ''' + partition_name = gen_unique_str() + new_partition_name = gen_unique_str() + new_tag = "new_tag" + index_params = get_simple_index_params + logging.getLogger().info(index_params) + status = connect.create_partition(ip_table, partition_name, tag) + status = connect.create_partition(ip_table, new_partition_name, new_tag) + status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) + status, ids = connect.add_vectors(ip_table, vectors, partition_tag=new_tag) + status = connect.create_index(partition_name, index_params) + status = connect.create_index(new_partition_name, index_params) + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == ip_table + assert result._index_type == IndexType.FLAT + status, result = connect.describe_index(new_partition_name) + logging.getLogger().info(result) + assert result._nlist == index_params["nlist"] + assert result._table_name == new_partition_name + assert result._index_type == index_params["index_type"] + def test_describe_and_drop_index_multi_tables(self, connect, get_simple_index_params): ''' target: test create, describe and drop index interface with multiple tables of IP @@ -849,6 +954,111 @@ class TestIndexIP: assert result._table_name == ip_table assert result._index_type == IndexType.FLAT + def test_drop_index_partition(self, connect, ip_table, get_simple_index_params): + ''' + target: test drop index interface + method: create table, create partition and add vectors in it, create index on table, call drop table index + expected: return code 0, and default index param + ''' + partition_name = gen_unique_str() + index_params = get_simple_index_params + status = connect.create_partition(ip_table, partition_name, tag) + status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) + status = connect.create_index(ip_table, index_params) + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + status = connect.drop_index(ip_table) + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == ip_table + assert result._index_type == IndexType.FLAT + + def test_drop_index_partition_A(self, connect, ip_table, get_simple_index_params): + ''' + target: test drop index interface + method: create table, create partition and add vectors in it, create index on partition, call drop table index + expected: return code 0, and default index param + ''' + partition_name = gen_unique_str() + index_params = get_simple_index_params + status = connect.create_partition(ip_table, partition_name, tag) + status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) + status = connect.create_index(partition_name, index_params) + assert status.OK() + status = connect.drop_index(ip_table) + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == ip_table + assert result._index_type == IndexType.FLAT + status, result = connect.describe_index(partition_name) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == partition_name + assert result._index_type == IndexType.FLAT + + def test_drop_index_partition_B(self, connect, ip_table, get_simple_index_params): + ''' + target: test drop index interface + method: create table, create partition and add vectors in it, create index on partition, call drop partition index + expected: return code 0, and default index param + ''' + partition_name = gen_unique_str() + index_params = get_simple_index_params + status = connect.create_partition(ip_table, partition_name, tag) + status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag) + status = connect.create_index(partition_name, index_params) + assert status.OK() + status = connect.drop_index(partition_name) + assert status.OK() + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == ip_table + assert result._index_type == IndexType.FLAT + status, result = connect.describe_index(partition_name) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == partition_name + assert result._index_type == IndexType.FLAT + + def test_drop_index_partition_C(self, connect, ip_table, get_simple_index_params): + ''' + target: test drop index interface + method: create table, create partitions and add vectors in it, create index on partitions, call drop partition index + expected: return code 0, and default index param + ''' + partition_name = gen_unique_str() + new_partition_name = gen_unique_str() + new_tag = "new_tag" + index_params = get_simple_index_params + status = connect.create_partition(ip_table, partition_name, tag) + status = connect.create_partition(ip_table, new_partition_name, new_tag) + status, ids = connect.add_vectors(ip_table, vectors) + status = connect.create_index(ip_table, index_params) + assert status.OK() + status = connect.drop_index(new_partition_name) + assert status.OK() + status, result = connect.describe_index(new_partition_name) + logging.getLogger().info(result) + assert result._nlist == 16384 + assert result._table_name == new_partition_name + assert result._index_type == IndexType.FLAT + status, result = connect.describe_index(partition_name) + logging.getLogger().info(result) + assert result._nlist == index_params["nlist"] + assert result._table_name == partition_name + assert result._index_type == index_params["index_type"] + status, result = connect.describe_index(ip_table) + logging.getLogger().info(result) + assert result._nlist == index_params["nlist"] + assert result._table_name == ip_table + assert result._index_type == index_params["index_type"] + def test_drop_index_repeatly(self, connect, ip_table, get_simple_index_params): ''' target: test drop index repeatly diff --git a/tests/milvus_python_test/test_mix.py b/tests/milvus_python_test/test_mix.py index f099db5c31..5ef9ba2cde 100644 --- a/tests/milvus_python_test/test_mix.py +++ b/tests/milvus_python_test/test_mix.py @@ -25,9 +25,8 @@ index_params = {'index_type': IndexType.IVFLAT, 'nlist': 16384} class TestMixBase: - # TODO: enable def test_search_during_createIndex(self, args): - loops = 100000 + loops = 10000 table = gen_unique_str() query_vecs = [vectors[0], vectors[1]] uri = "tcp://%s:%s" % (args["ip"], args["port"]) diff --git a/tests/milvus_python_test/test_partition.py b/tests/milvus_python_test/test_partition.py new file mode 100644 index 0000000000..cbb0b5bc8e --- /dev/null +++ b/tests/milvus_python_test/test_partition.py @@ -0,0 +1,431 @@ +import time +import random +import pdb +import threading +import logging +from multiprocessing import Pool, Process +import pytest +from milvus import Milvus, IndexType, MetricType +from utils import * + + +dim = 128 +index_file_size = 10 +table_id = "test_add" +ADD_TIMEOUT = 60 +nprobe = 1 +epsilon = 0.0001 +tag = "1970-01-01" + + +class TestCreateBase: + + """ + ****************************************************************** + The following cases are used to test `create_partition` function + ****************************************************************** + """ + def test_create_partition(self, connect, table): + ''' + target: test create partition, check status returned + method: call function: create_partition + expected: status ok + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + assert status.OK() + + def test_create_partition_repeat(self, connect, table): + ''' + target: test create partition, check status returned + method: call function: create_partition + expected: status ok + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + status = connect.create_partition(table, partition_name, tag) + assert not status.OK() + + def test_create_partition_recursively(self, connect, table): + ''' + target: test create partition, and create partition in parent partition, check status returned + method: call function: create_partition + expected: status not ok + ''' + partition_name = gen_unique_str() + new_partition_name = gen_unique_str() + new_tag = "new_tag" + status = connect.create_partition(table, partition_name, tag) + status = connect.create_partition(partition_name, new_partition_name, new_tag) + assert not status.OK() + + def test_create_partition_table_not_existed(self, connect): + ''' + target: test create partition, its owner table name not existed in db, check status returned + method: call function: create_partition + expected: status not ok + ''' + table_name = gen_unique_str() + partition_name = gen_unique_str() + status = connect.create_partition(table_name, partition_name, tag) + assert not status.OK() + + def test_create_partition_partition_name_existed(self, connect, table): + ''' + target: test create partition, and create the same partition again, check status returned + method: call function: create_partition + expected: status not ok + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + assert status.OK() + tag_new = "tag_new" + status = connect.create_partition(table, partition_name, tag_new) + assert not status.OK() + + def test_create_partition_partition_name_equals_table(self, connect, table): + ''' + target: test create partition, the partition equals to table, check status returned + method: call function: create_partition + expected: status not ok + ''' + status = connect.create_partition(table, table, tag) + assert not status.OK() + + def test_create_partition_partition_name_None(self, connect, table): + ''' + target: test create partition, partition name set None, check status returned + method: call function: create_partition + expected: status not ok + ''' + partition_name = None + status = connect.create_partition(table, partition_name, tag) + assert not status.OK() + + def test_create_partition_tag_name_None(self, connect, table): + ''' + target: test create partition, tag name set None, check status returned + method: call function: create_partition + expected: status ok + ''' + tag_name = None + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag_name) + assert not status.OK() + + def test_create_different_partition_tag_name_existed(self, connect, table): + ''' + target: test create partition, and create the same partition tag again, check status returned + method: call function: create_partition with the same tag name + expected: status not ok + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + assert status.OK() + new_partition_name = gen_unique_str() + status = connect.create_partition(table, new_partition_name, tag) + assert not status.OK() + + def test_create_partition_add_vectors(self, connect, table): + ''' + target: test create partition, and insert vectors, check status returned + method: call function: create_partition + expected: status ok + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + assert status.OK() + nq = 100 + vectors = gen_vectors(nq, dim) + ids = [i for i in range(nq)] + status, ids = connect.insert(table, vectors, ids) + assert status.OK() + + def test_create_partition_insert_with_tag(self, connect, table): + ''' + target: test create partition, and insert vectors, check status returned + method: call function: create_partition + expected: status ok + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + assert status.OK() + nq = 100 + vectors = gen_vectors(nq, dim) + ids = [i for i in range(nq)] + status, ids = connect.insert(table, vectors, ids, partition_tag=tag) + assert status.OK() + + def test_create_partition_insert_with_tag_not_existed(self, connect, table): + ''' + target: test create partition, and insert vectors, check status returned + method: call function: create_partition + expected: status not ok + ''' + tag_new = "tag_new" + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + assert status.OK() + nq = 100 + vectors = gen_vectors(nq, dim) + ids = [i for i in range(nq)] + status, ids = connect.insert(table, vectors, ids, partition_tag=tag_new) + assert not status.OK() + + def test_create_partition_insert_same_tags(self, connect, table): + ''' + target: test create partition, and insert vectors, check status returned + method: call function: create_partition + expected: status ok + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + assert status.OK() + nq = 100 + vectors = gen_vectors(nq, dim) + ids = [i for i in range(nq)] + status, ids = connect.insert(table, vectors, ids, partition_tag=tag) + ids = [(i+100) for i in range(nq)] + status, ids = connect.insert(table, vectors, ids, partition_tag=tag) + assert status.OK() + time.sleep(1) + status, res = connect.get_table_row_count(partition_name) + assert res == nq * 2 + + def test_create_partition_insert_same_tags_two_tables(self, connect, table): + ''' + target: test create two partitions, and insert vectors with the same tag to each table, check status returned + method: call function: create_partition + expected: status ok, table length is correct + ''' + partition_name = gen_unique_str() + table_new = gen_unique_str() + new_partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + assert status.OK() + param = {'table_name': table_new, + 'dimension': dim, + 'index_file_size': index_file_size, + 'metric_type': MetricType.L2} + status = connect.create_table(param) + status = connect.create_partition(table_new, new_partition_name, tag) + assert status.OK() + nq = 100 + vectors = gen_vectors(nq, dim) + ids = [i for i in range(nq)] + status, ids = connect.insert(table, vectors, ids, partition_tag=tag) + ids = [(i+100) for i in range(nq)] + status, ids = connect.insert(table_new, vectors, ids, partition_tag=tag) + assert status.OK() + time.sleep(1) + status, res = connect.get_table_row_count(new_partition_name) + assert res == nq + + +class TestShowBase: + + """ + ****************************************************************** + The following cases are used to test `show_partitions` function + ****************************************************************** + """ + def test_show_partitions(self, connect, table): + ''' + target: test show partitions, check status and partitions returned + method: create partition first, then call function: show_partitions + expected: status ok, partition correct + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + status, res = connect.show_partitions(table) + assert status.OK() + + def test_show_partitions_no_partition(self, connect, table): + ''' + target: test show partitions with table name, check status and partitions returned + method: call function: show_partitions + expected: status ok, partitions correct + ''' + partition_name = gen_unique_str() + status, res = connect.show_partitions(table) + assert status.OK() + + def test_show_partitions_no_partition_recursive(self, connect, table): + ''' + target: test show partitions with partition name, check status and partitions returned + method: call function: show_partitions + expected: status ok, no partitions + ''' + partition_name = gen_unique_str() + status, res = connect.show_partitions(partition_name) + assert status.OK() + assert len(res) == 0 + + def test_show_multi_partitions(self, connect, table): + ''' + target: test show partitions, check status and partitions returned + method: create partitions first, then call function: show_partitions + expected: status ok, partitions correct + ''' + partition_name = gen_unique_str() + new_partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + status = connect.create_partition(table, new_partition_name, tag) + status, res = connect.show_partitions(table) + assert status.OK() + + +class TestDropBase: + + """ + ****************************************************************** + The following cases are used to test `drop_partition` function + ****************************************************************** + """ + def test_drop_partition(self, connect, table): + ''' + target: test drop partition, check status and partition if existed + method: create partitions first, then call function: drop_partition + expected: status ok, no partitions in db + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + status = connect.drop_partition(table, tag) + assert status.OK() + # check if the partition existed + status, res = connect.show_partitions(table) + assert partition_name not in res + + def test_drop_partition_tag_not_existed(self, connect, table): + ''' + target: test drop partition, but tag not existed + method: create partitions first, then call function: drop_partition + expected: status not ok + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + new_tag = "new_tag" + status = connect.drop_partition(table, new_tag) + assert not status.OK() + + def test_drop_partition_tag_not_existed_A(self, connect, table): + ''' + target: test drop partition, but table not existed + method: create partitions first, then call function: drop_partition + expected: status not ok + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + new_table = gen_unique_str() + status = connect.drop_partition(new_table, tag) + assert not status.OK() + + def test_drop_partition_repeatedly(self, connect, table): + ''' + target: test drop partition twice, check status and partition if existed + method: create partitions first, then call function: drop_partition + expected: status not ok, no partitions in db + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + status = connect.drop_partition(table, tag) + status = connect.drop_partition(table, tag) + time.sleep(2) + assert not status.OK() + status, res = connect.show_partitions(table) + assert partition_name not in res + + def test_drop_partition_create(self, connect, table): + ''' + target: test drop partition, and create again, check status + method: create partitions first, then call function: drop_partition, create_partition + expected: status not ok, partition in db + ''' + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + status = connect.drop_partition(table, tag) + time.sleep(2) + status = connect.create_partition(table, partition_name, tag) + assert status.OK() + status, res = connect.show_partitions(table) + assert partition_name == res[0].partition_name + + +class TestNameInvalid(object): + @pytest.fixture( + scope="function", + params=gen_invalid_table_names() + ) + def get_partition_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_table_names() + ) + def get_tag_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_table_names() + ) + def get_table_name(self, request): + yield request.param + + def test_create_partition_with_invalid_partition_name(self, connect, table, get_partition_name): + ''' + target: test create partition, with invalid partition name, check status returned + method: call function: create_partition + expected: status not ok + ''' + partition_name = get_partition_name + status = connect.create_partition(table, partition_name, tag) + assert not status.OK() + + def test_create_partition_with_invalid_tag_name(self, connect, table): + ''' + target: test create partition, with invalid partition name, check status returned + method: call function: create_partition + expected: status not ok + ''' + tag_name = " " + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag_name) + assert not status.OK() + + def test_drop_partition_with_invalid_table_name(self, connect, table, get_table_name): + ''' + target: test drop partition, with invalid table name, check status returned + method: call function: drop_partition + expected: status not ok + ''' + table_name = get_table_name + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + status = connect.drop_partition(table_name, tag) + assert not status.OK() + + def test_drop_partition_with_invalid_tag_name(self, connect, table, get_tag_name): + ''' + target: test drop partition, with invalid tag name, check status returned + method: call function: drop_partition + expected: status not ok + ''' + tag_name = get_tag_name + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + status = connect.drop_partition(table, tag_name) + assert not status.OK() + + def test_show_partitions_with_invalid_table_name(self, connect, table, get_table_name): + ''' + target: test show partitions, with invalid table name, check status returned + method: call function: show_partitions + expected: status not ok + ''' + table_name = get_table_name + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + status, res = connect.show_partitions(table_name) + assert not status.OK() \ No newline at end of file diff --git a/tests/milvus_python_test/test_search_vectors.py b/tests/milvus_python_test/test_search_vectors.py index 10892d6de3..e0b1bc09ea 100644 --- a/tests/milvus_python_test/test_search_vectors.py +++ b/tests/milvus_python_test/test_search_vectors.py @@ -16,8 +16,9 @@ add_interval_time = 2 vectors = gen_vectors(100, dim) # vectors /= numpy.linalg.norm(vectors) # vectors = vectors.tolist() -nrpobe = 1 +nprobe = 1 epsilon = 0.001 +tag = "1970-01-01" class TestSearchBase: @@ -49,6 +50,15 @@ class TestSearchBase: pytest.skip("sq8h not support in open source") return request.param + @pytest.fixture( + scope="function", + params=gen_simple_index_params() + ) + def get_simple_index_params(self, request, args): + if "internal" not in args: + if request.param["index_type"] == IndexType.IVF_SQ8H: + pytest.skip("sq8h not support in open source") + return request.param """ generate top-k params """ @@ -70,7 +80,7 @@ class TestSearchBase: query_vec = [vectors[0]] top_k = get_top_k nprobe = 1 - status, result = connect.search_vectors(table, top_k, nrpobe, query_vec) + status, result = connect.search_vectors(table, top_k, nprobe, query_vec) if top_k <= 2048: assert status.OK() assert len(result[0]) == min(len(vectors), top_k) @@ -85,7 +95,6 @@ class TestSearchBase: method: search with the given vectors, check the result expected: search status ok, and the length of the result is top_k ''' - index_params = get_index_params logging.getLogger().info(index_params) vectors, ids = self.init_data(connect, table) @@ -93,7 +102,7 @@ class TestSearchBase: query_vec = [vectors[0]] top_k = 10 nprobe = 1 - status, result = connect.search_vectors(table, top_k, nrpobe, query_vec) + status, result = connect.search_vectors(table, top_k, nprobe, query_vec) logging.getLogger().info(result) if top_k <= 1024: assert status.OK() @@ -103,6 +112,160 @@ class TestSearchBase: else: assert not status.OK() + def test_search_l2_index_params_partition(self, connect, table, get_simple_index_params): + ''' + target: test basic search fuction, all the search params is corrent, test all index params, and build + method: add vectors into table, search with the given vectors, check the result + expected: search status ok, and the length of the result is top_k, search table with partition tag return empty + ''' + index_params = get_simple_index_params + logging.getLogger().info(index_params) + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + vectors, ids = self.init_data(connect, table) + status = connect.create_index(table, index_params) + query_vec = [vectors[0]] + top_k = 10 + nprobe = 1 + status, result = connect.search_vectors(table, top_k, nprobe, query_vec) + logging.getLogger().info(result) + assert status.OK() + assert len(result[0]) == min(len(vectors), top_k) + assert check_result(result[0], ids[0]) + assert result[0][0].distance <= epsilon + status, result = connect.search_vectors(table, top_k, nprobe, query_vec, partition_tags=[tag]) + logging.getLogger().info(result) + assert status.OK() + assert len(result) == 0 + + def test_search_l2_index_params_partition_A(self, connect, table, get_simple_index_params): + ''' + target: test basic search fuction, all the search params is corrent, test all index params, and build + method: search partition with the given vectors, check the result + expected: search status ok, and the length of the result is 0 + ''' + index_params = get_simple_index_params + logging.getLogger().info(index_params) + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + vectors, ids = self.init_data(connect, table) + status = connect.create_index(table, index_params) + query_vec = [vectors[0]] + top_k = 10 + nprobe = 1 + status, result = connect.search_vectors(partition_name, top_k, nprobe, query_vec, partition_tags=[tag]) + logging.getLogger().info(result) + assert status.OK() + assert len(result) == 0 + + def test_search_l2_index_params_partition_B(self, connect, table, get_simple_index_params): + ''' + target: test basic search fuction, all the search params is corrent, test all index params, and build + method: search with the given vectors, check the result + expected: search status ok, and the length of the result is top_k + ''' + index_params = get_simple_index_params + logging.getLogger().info(index_params) + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + vectors, ids = self.init_data(connect, partition_name) + status = connect.create_index(table, index_params) + query_vec = [vectors[0]] + top_k = 10 + nprobe = 1 + status, result = connect.search_vectors(table, top_k, nprobe, query_vec) + logging.getLogger().info(result) + assert status.OK() + assert len(result[0]) == min(len(vectors), top_k) + assert check_result(result[0], ids[0]) + assert result[0][0].distance <= epsilon + status, result = connect.search_vectors(table, top_k, nprobe, query_vec, partition_tags=[tag]) + logging.getLogger().info(result) + assert status.OK() + assert len(result[0]) == min(len(vectors), top_k) + assert check_result(result[0], ids[0]) + assert result[0][0].distance <= epsilon + status, result = connect.search_vectors(partition_name, top_k, nprobe, query_vec, partition_tags=[tag]) + logging.getLogger().info(result) + assert status.OK() + assert len(result) == 0 + + def test_search_l2_index_params_partition_C(self, connect, table, get_simple_index_params): + ''' + target: test basic search fuction, all the search params is corrent, test all index params, and build + method: search with the given vectors and tags (one of the tags not existed in table), check the result + expected: search status ok, and the length of the result is top_k + ''' + index_params = get_simple_index_params + logging.getLogger().info(index_params) + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + vectors, ids = self.init_data(connect, partition_name) + status = connect.create_index(table, index_params) + query_vec = [vectors[0]] + top_k = 10 + nprobe = 1 + status, result = connect.search_vectors(table, top_k, nprobe, query_vec, partition_tags=[tag, "new_tag"]) + logging.getLogger().info(result) + assert status.OK() + assert len(result[0]) == min(len(vectors), top_k) + assert check_result(result[0], ids[0]) + assert result[0][0].distance <= epsilon + + def test_search_l2_index_params_partition_D(self, connect, table, get_simple_index_params): + ''' + target: test basic search fuction, all the search params is corrent, test all index params, and build + method: search with the given vectors and tag (tag name not existed in table), check the result + expected: search status ok, and the length of the result is top_k + ''' + index_params = get_simple_index_params + logging.getLogger().info(index_params) + partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + vectors, ids = self.init_data(connect, partition_name) + status = connect.create_index(table, index_params) + query_vec = [vectors[0]] + top_k = 10 + nprobe = 1 + status, result = connect.search_vectors(table, top_k, nprobe, query_vec, partition_tags=["new_tag"]) + logging.getLogger().info(result) + assert status.OK() + assert len(result) == 0 + + def test_search_l2_index_params_partition_E(self, connect, table, get_simple_index_params): + ''' + target: test basic search fuction, all the search params is corrent, test all index params, and build + method: search table with the given vectors and tags, check the result + expected: search status ok, and the length of the result is top_k + ''' + new_tag = "new_tag" + index_params = get_simple_index_params + logging.getLogger().info(index_params) + partition_name = gen_unique_str() + new_partition_name = gen_unique_str() + status = connect.create_partition(table, partition_name, tag) + status = connect.create_partition(table, new_partition_name, new_tag) + vectors, ids = self.init_data(connect, partition_name) + new_vectors, new_ids = self.init_data(connect, new_partition_name, nb=1000) + status = connect.create_index(table, index_params) + query_vec = [vectors[0], new_vectors[0]] + top_k = 10 + nprobe = 1 + status, result = connect.search_vectors(table, top_k, nprobe, query_vec, partition_tags=[tag, new_tag]) + logging.getLogger().info(result) + assert status.OK() + assert len(result[0]) == min(len(vectors), top_k) + assert check_result(result[0], ids[0]) + assert check_result(result[1], new_ids[0]) + assert result[0][0].distance <= epsilon + assert result[1][0].distance <= epsilon + status, result = connect.search_vectors(table, top_k, nprobe, query_vec, partition_tags=[new_tag]) + logging.getLogger().info(result) + assert status.OK() + assert len(result[0]) == min(len(vectors), top_k) + assert check_result(result[1], new_ids[0]) + assert result[1][0].distance <= epsilon + def test_search_ip_index_params(self, connect, ip_table, get_index_params): ''' target: test basic search fuction, all the search params is corrent, test all index params, and build @@ -117,7 +280,7 @@ class TestSearchBase: query_vec = [vectors[0]] top_k = 10 nprobe = 1 - status, result = connect.search_vectors(ip_table, top_k, nrpobe, query_vec) + status, result = connect.search_vectors(ip_table, top_k, nprobe, query_vec) logging.getLogger().info(result) if top_k <= 1024: @@ -128,6 +291,59 @@ class TestSearchBase: else: assert not status.OK() + def test_search_ip_index_params_partition(self, connect, ip_table, get_simple_index_params): + ''' + target: test basic search fuction, all the search params is corrent, test all index params, and build + method: search with the given vectors, check the result + expected: search status ok, and the length of the result is top_k + ''' + index_params = get_simple_index_params + logging.getLogger().info(index_params) + partition_name = gen_unique_str() + status = connect.create_partition(ip_table, partition_name, tag) + vectors, ids = self.init_data(connect, ip_table) + status = connect.create_index(ip_table, index_params) + query_vec = [vectors[0]] + top_k = 10 + nprobe = 1 + status, result = connect.search_vectors(ip_table, top_k, nprobe, query_vec) + logging.getLogger().info(result) + assert status.OK() + assert len(result[0]) == min(len(vectors), top_k) + assert check_result(result[0], ids[0]) + assert abs(result[0][0].distance - numpy.inner(numpy.array(query_vec[0]), numpy.array(query_vec[0]))) <= gen_inaccuracy(result[0][0].distance) + status, result = connect.search_vectors(ip_table, top_k, nprobe, query_vec, partition_tags=[tag]) + logging.getLogger().info(result) + assert status.OK() + assert len(result) == 0 + + def test_search_ip_index_params_partition_A(self, connect, ip_table, get_simple_index_params): + ''' + target: test basic search fuction, all the search params is corrent, test all index params, and build + method: search with the given vectors and tag, check the result + expected: search status ok, and the length of the result is top_k + ''' + index_params = get_simple_index_params + logging.getLogger().info(index_params) + partition_name = gen_unique_str() + status = connect.create_partition(ip_table, partition_name, tag) + vectors, ids = self.init_data(connect, partition_name) + status = connect.create_index(ip_table, index_params) + query_vec = [vectors[0]] + top_k = 10 + nprobe = 1 + status, result = connect.search_vectors(ip_table, top_k, nprobe, query_vec, partition_tags=[tag]) + logging.getLogger().info(result) + assert status.OK() + assert len(result[0]) == min(len(vectors), top_k) + assert check_result(result[0], ids[0]) + assert abs(result[0][0].distance - numpy.inner(numpy.array(query_vec[0]), numpy.array(query_vec[0]))) <= gen_inaccuracy(result[0][0].distance) + status, result = connect.search_vectors(partition_name, top_k, nprobe, query_vec) + logging.getLogger().info(result) + assert status.OK() + assert len(result[0]) == min(len(vectors), top_k) + assert check_result(result[0], ids[0]) + @pytest.mark.level(2) def test_search_vectors_without_connect(self, dis_connect, table): ''' @@ -518,6 +734,14 @@ class TestSearchParamsInvalid(object): status, result = connect.search_vectors(table_name, top_k, nprobe, query_vecs) assert not status.OK() + @pytest.mark.level(1) + def test_search_with_invalid_tag_format(self, connect, table): + top_k = 1 + nprobe = 1 + query_vecs = gen_vectors(1, dim) + with pytest.raises(Exception) as e: + status, result = connect.search_vectors(table_name, top_k, nprobe, query_vecs, partition_tags="tag") + """ Test search table with invalid top-k """ @@ -574,7 +798,7 @@ class TestSearchParamsInvalid(object): yield request.param @pytest.mark.level(1) - def test_search_with_invalid_nrpobe(self, connect, table, get_nprobes): + def test_search_with_invalid_nprobe(self, connect, table, get_nprobes): ''' target: test search fuction, with the wrong top_k method: search with top_k @@ -592,7 +816,7 @@ class TestSearchParamsInvalid(object): status, result = connect.search_vectors(table, top_k, nprobe, query_vecs) @pytest.mark.level(2) - def test_search_with_invalid_nrpobe_ip(self, connect, ip_table, get_nprobes): + def test_search_with_invalid_nprobe_ip(self, connect, ip_table, get_nprobes): ''' target: test search fuction, with the wrong top_k method: search with top_k diff --git a/tests/milvus_python_test/test_table.py b/tests/milvus_python_test/test_table.py index 6af38bac15..40b0850859 100644 --- a/tests/milvus_python_test/test_table.py +++ b/tests/milvus_python_test/test_table.py @@ -297,7 +297,7 @@ class TestTable: ''' table_name = gen_unique_str("test_table") status = connect.delete_table(table_name) - assert not status.code==0 + assert not status.OK() def test_delete_table_repeatedly(self, connect): ''' diff --git a/tests/milvus_python_test/test_table_count.py b/tests/milvus_python_test/test_table_count.py index 4e8a780c62..77780c8faa 100644 --- a/tests/milvus_python_test/test_table_count.py +++ b/tests/milvus_python_test/test_table_count.py @@ -13,8 +13,8 @@ from milvus import IndexType, MetricType dim = 128 index_file_size = 10 -add_time_interval = 5 - +add_time_interval = 3 +tag = "1970-01-01" class TestTableCount: """ @@ -58,6 +58,90 @@ class TestTableCount: status, res = connect.get_table_row_count(table) assert res == nb + def test_table_rows_count_partition(self, connect, table, add_vectors_nb): + ''' + target: test table rows_count is correct or not + method: create table, create partition and add vectors in it, + assert the value returned by get_table_row_count method is equal to length of vectors + expected: the count is equal to the length of vectors + ''' + nb = add_vectors_nb + partition_name = gen_unique_str() + vectors = gen_vectors(nb, dim) + status = connect.create_partition(table, partition_name, tag) + assert status.OK() + res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag) + time.sleep(add_time_interval) + status, res = connect.get_table_row_count(table) + assert res == nb + + def test_table_rows_count_multi_partitions_A(self, connect, table, add_vectors_nb): + ''' + target: test table rows_count is correct or not + method: create table, create partitions and add vectors in it, + assert the value returned by get_table_row_count method is equal to length of vectors + expected: the count is equal to the length of vectors + ''' + new_tag = "new_tag" + nb = add_vectors_nb + partition_name = gen_unique_str() + new_partition_name = gen_unique_str() + vectors = gen_vectors(nb, dim) + status = connect.create_partition(table, partition_name, tag) + status = connect.create_partition(table, new_partition_name, new_tag) + assert status.OK() + res = connect.add_vectors(table_name=table, records=vectors) + time.sleep(add_time_interval) + status, res = connect.get_table_row_count(table) + assert res == nb + + def test_table_rows_count_multi_partitions_B(self, connect, table, add_vectors_nb): + ''' + target: test table rows_count is correct or not + method: create table, create partitions and add vectors in one of the partitions, + assert the value returned by get_table_row_count method is equal to length of vectors + expected: the count is equal to the length of vectors + ''' + new_tag = "new_tag" + nb = add_vectors_nb + partition_name = gen_unique_str() + new_partition_name = gen_unique_str() + vectors = gen_vectors(nb, dim) + status = connect.create_partition(table, partition_name, tag) + status = connect.create_partition(table, new_partition_name, new_tag) + assert status.OK() + res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag) + time.sleep(add_time_interval) + status, res = connect.get_table_row_count(partition_name) + assert res == nb + status, res = connect.get_table_row_count(new_partition_name) + assert res == 0 + + def test_table_rows_count_multi_partitions_C(self, connect, table, add_vectors_nb): + ''' + target: test table rows_count is correct or not + method: create table, create partitions and add vectors in one of the partitions, + assert the value returned by get_table_row_count method is equal to length of vectors + expected: the table count is equal to the length of vectors + ''' + new_tag = "new_tag" + nb = add_vectors_nb + partition_name = gen_unique_str() + new_partition_name = gen_unique_str() + vectors = gen_vectors(nb, dim) + status = connect.create_partition(table, partition_name, tag) + status = connect.create_partition(table, new_partition_name, new_tag) + assert status.OK() + res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag) + res = connect.add_vectors(table_name=table, records=vectors, partition_tag=new_tag) + time.sleep(add_time_interval) + status, res = connect.get_table_row_count(partition_name) + assert res == nb + status, res = connect.get_table_row_count(new_partition_name) + assert res == nb + status, res = connect.get_table_row_count(table) + assert res == nb * 2 + def test_table_rows_count_after_index_created(self, connect, table, get_simple_index_params): ''' target: test get_table_row_count, after index have been created