From 3fda33ac0bba66e9eac5b1162c860fcf0fbe299e Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Tue, 2 Jul 2019 19:36:06 +0800 Subject: [PATCH 1/6] alter nprobe Former-commit-id: 21cc3f6523580303bf702f7e616bcee7416a053f --- cpp/conf/server_config.yaml | 7 +++++-- cpp/src/wrapper/Index.cpp | 31 ++++++++++++++++++++++++++++++- cpp/src/wrapper/Operand.cpp | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 0053ba365a..359f8d86bf 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -5,7 +5,7 @@ server_config: mode: single # milvus deployment type: single, cluster db_config: - db_path: /opt/milvus # milvus data storage path + db_path: /tmp/milvus # milvus data storage path db_backend_url: http://127.0.0.1 # meta database uri index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB @@ -22,7 +22,10 @@ metric_config: license_config: # license configure - license_path: "/opt/milvus/system.license" # license file path + license_path: "/tmp/milvus/system.license" # license file path cache_config: # cache configure cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory + +engine_config: + nprobe: 3000 \ No newline at end of file diff --git a/cpp/src/wrapper/Index.cpp b/cpp/src/wrapper/Index.cpp index c306b65e1a..b051bfb07e 100644 --- a/cpp/src/wrapper/Index.cpp +++ b/cpp/src/wrapper/Index.cpp @@ -14,6 +14,8 @@ #include "Index.h" #include "faiss/index_io.h" #include "faiss/IndexIVF.h" +#include +#include "server/ServerConfig.h" namespace zilliz { namespace milvus { @@ -23,6 +25,32 @@ using std::string; using std::unordered_map; using std::vector; +class Nprobe { + public: + static Nprobe &GetInstance() { + static Nprobe instance; + return instance; + } + + void SelectNprobe() { + using namespace zilliz::milvus::server; + ServerConfig &config = ServerConfig::GetInstance(); + ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); + nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000); + } + + size_t GetNprobe() { + return nprobe_; + } + + private: + Nprobe() : nprobe_(1000) { SelectNprobe(); } + + private: + size_t nprobe_; +}; + + Index::Index(const std::shared_ptr &raw_index) { index_ = raw_index; dim = index_->d; @@ -57,7 +85,8 @@ bool Index::add_with_ids(idx_t n, const float *xdata, const long *xids) { bool Index::search(idx_t n, const float *data, idx_t k, float *distances, long *labels) const { try { if(auto ivf_index = std::dynamic_pointer_cast(index_)) { - ivf_index->nprobe = 100; + ivf_index->nprobe = Nprobe::GetInstance().GetNprobe(); + std::cout << "nprobe = " << ivf_index->nprobe << std::endl; } index_->search(n, data, k, distances, labels); } diff --git a/cpp/src/wrapper/Operand.cpp b/cpp/src/wrapper/Operand.cpp index 30e31067fd..25341676a6 100644 --- a/cpp/src/wrapper/Operand.cpp +++ b/cpp/src/wrapper/Operand.cpp @@ -39,7 +39,7 @@ string Operand::get_index_type(const int &nb) { } case IVF: { index_str += (ncent != 0 ? index_type + std::to_string(ncent) : - index_type + std::to_string(int(nb / 1000000.0 * 1638))); + index_type + std::to_string(int(nb / 1000000.0 * 16384))); break; } case IDMAP: { From c480cb8b7f38d581287f72ad52cf77864b609ed5 Mon Sep 17 00:00:00 2001 From: jinhai Date: Tue, 2 Jul 2019 20:06:00 +0800 Subject: [PATCH 2/6] Fix typo Former-commit-id: 1bccdd0999347c1fa76bdef446e9a7a2109ef66a --- cpp/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index bfdafcc1c3..ec536c2ee2 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -26,7 +26,7 @@ using namespace zilliz::milvus; int main(int argc, char *argv[]) { - std::cout << std::endl << "Welcome to use Milvus by Zillz!" << std::endl; + std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl; std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl; signal(SIGINT, server::SignalUtil::HandleSignal); From 87af3d632de89fc976d2a6b7045b72ac1f1ca010 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Tue, 2 Jul 2019 20:13:10 +0800 Subject: [PATCH 3/6] add engine config Former-commit-id: 326501e3adc70b88fe3b8304ba39920fb11cf2d0 --- cpp/src/server/ServerConfig.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index f337275a46..412581bc1f 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -43,6 +43,9 @@ static const std::string CONFIG_METRIC_COLLECTOR = "collector"; static const std::string CONFIG_PROMETHEUS = "prometheus_config"; static const std::string CONFIG_METRIC_PROMETHEUS_PORT = "port"; +static const std::string CONFIG_ENGINE = "engine_config"; +static const std::string CONFIG_NPROBE = "nprobe"; + class ServerConfig { public: static ServerConfig &GetInstance(); From 380363f477b7ddb7f3181fa56369bb516ebbc04d Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Tue, 2 Jul 2019 20:24:41 +0800 Subject: [PATCH 4/6] update Former-commit-id: 99f1f53c6f2b1775f78acf0ba5560a112c68140b --- cpp/src/wrapper/Index.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/src/wrapper/Index.cpp b/cpp/src/wrapper/Index.cpp index b051bfb07e..57c462a201 100644 --- a/cpp/src/wrapper/Index.cpp +++ b/cpp/src/wrapper/Index.cpp @@ -14,7 +14,7 @@ #include "Index.h" #include "faiss/index_io.h" #include "faiss/IndexIVF.h" -#include +#include "faiss/IVFlib.h" #include "server/ServerConfig.h" namespace zilliz { @@ -86,7 +86,6 @@ bool Index::search(idx_t n, const float *data, idx_t k, float *distances, long * try { if(auto ivf_index = std::dynamic_pointer_cast(index_)) { ivf_index->nprobe = Nprobe::GetInstance().GetNprobe(); - std::cout << "nprobe = " << ivf_index->nprobe << std::endl; } index_->search(n, data, k, distances, labels); } From 4aabe33207c68c84e1c94af7d59033eea935b3da Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 3 Jul 2019 11:09:17 +0800 Subject: [PATCH 5/6] update IVF nprobe Former-commit-id: 41aa77c14de1db37cfb1a177ba223de90829e003 --- cpp/conf/server_config.template | 5 ++++- cpp/src/db/EngineFactory.cpp | 34 +++++++++++++++++++++-------- cpp/src/db/ExecutionEngine.h | 2 ++ cpp/src/db/FaissExecutionEngine.cpp | 30 ++++++++++++++++++++++++- cpp/src/db/FaissExecutionEngine.h | 9 ++++---- cpp/src/wrapper/Index.cpp | 29 ------------------------ 6 files changed, 65 insertions(+), 44 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index c2ed775601..0383e00b53 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -30,4 +30,7 @@ license_config: # license configure license_path: "@MILVUS_DB_PATH@/system.license" # license file path cache_config: # cache configure - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory + +engine_config: + nprobe: 10 \ No newline at end of file diff --git a/cpp/src/db/EngineFactory.cpp b/cpp/src/db/EngineFactory.cpp index 26ef639c88..bacce70ce4 100644 --- a/cpp/src/db/EngineFactory.cpp +++ b/cpp/src/db/EngineFactory.cpp @@ -7,23 +7,39 @@ #include "FaissExecutionEngine.h" #include "Log.h" + namespace zilliz { namespace milvus { namespace engine { ExecutionEnginePtr EngineFactory::Build(uint16_t dimension, - const std::string& location, - EngineType type) { - switch(type) { - case EngineType::FAISS_IDMAP: - return ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat")); - case EngineType::FAISS_IVFFLAT: - return ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat")); - default: - ENGINE_LOG_ERROR << "Unsupportted engine type"; + const std::string &location, + EngineType type) { + + ExecutionEnginePtr execution_engine_ptr; + + switch (type) { + case EngineType::FAISS_IDMAP: { + execution_engine_ptr = + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat")); + break; + } + + case EngineType::FAISS_IVFFLAT: { + execution_engine_ptr = + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat")); + break; + } + + default: { + ENGINE_LOG_ERROR << "Unsupported engine type"; return nullptr; + } } + + execution_engine_ptr->Init(); + return execution_engine_ptr; } } diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/ExecutionEngine.h index d2b4d01e67..f8c05f6f9d 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/ExecutionEngine.h @@ -50,6 +50,8 @@ public: virtual std::shared_ptr BuildIndex(const std::string&) = 0; virtual Status Cache() = 0; + + virtual Status Init() = 0; }; using ExecutionEnginePtr = std::shared_ptr; diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 9dfdd978c3..20bd530e78 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -13,6 +13,7 @@ #include #include #include +#include "faiss/IndexIVF.h" #include "metrics/Metrics.h" @@ -135,7 +136,16 @@ Status FaissExecutionEngine::Search(long n, float *distances, long *labels) const { auto start_time = METRICS_NOW_TIME; - pIndex_->search(n, data, k, distances, labels); + + std::shared_ptr ivf_index = std::dynamic_pointer_cast(pIndex_); + if(ivf_index) { + ENGINE_LOG_DEBUG << "Index type: IVFFLAT nProbe: " << nprobe_; + ivf_index->nprobe = nprobe_; + ivf_index->search(n, data, k, distances, labels); + } else { + pIndex_->search(n, data, k, distances, labels); + } + auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time,end_time); server::Metrics::GetInstance().QueryIndexTypePerSecondSet(build_index_type_, double(n)/double(total_time)); @@ -149,6 +159,24 @@ Status FaissExecutionEngine::Cache() { return Status::OK(); } +Status FaissExecutionEngine::Init() { + + if(build_index_type_ == "IVF") { + + using namespace zilliz::milvus::server; + ServerConfig &config = ServerConfig::GetInstance(); + ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); + nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000); + + } else if(build_index_type_ == "IDMap") { + ; + } else { + return Status::Error("Wrong index type: ", build_index_type_); + } + + return Status::OK(); +} + } // namespace engine } // namespace milvus diff --git a/cpp/src/db/FaissExecutionEngine.h b/cpp/src/db/FaissExecutionEngine.h index 5667df34ea..f9f37ad978 100644 --- a/cpp/src/db/FaissExecutionEngine.h +++ b/cpp/src/db/FaissExecutionEngine.h @@ -6,14 +6,11 @@ #pragma once #include "ExecutionEngine.h" +#include "faiss/Index.h" #include #include -namespace faiss { - class Index; -} - namespace zilliz { namespace milvus { namespace engine { @@ -58,12 +55,16 @@ public: Status Cache() override; + Status Init() override; + protected: std::shared_ptr pIndex_; std::string location_; std::string build_index_type_; std::string raw_index_type_; + + size_t nprobe_ = 0; }; diff --git a/cpp/src/wrapper/Index.cpp b/cpp/src/wrapper/Index.cpp index 57c462a201..18e20d830a 100644 --- a/cpp/src/wrapper/Index.cpp +++ b/cpp/src/wrapper/Index.cpp @@ -25,32 +25,6 @@ using std::string; using std::unordered_map; using std::vector; -class Nprobe { - public: - static Nprobe &GetInstance() { - static Nprobe instance; - return instance; - } - - void SelectNprobe() { - using namespace zilliz::milvus::server; - ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); - nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000); - } - - size_t GetNprobe() { - return nprobe_; - } - - private: - Nprobe() : nprobe_(1000) { SelectNprobe(); } - - private: - size_t nprobe_; -}; - - Index::Index(const std::shared_ptr &raw_index) { index_ = raw_index; dim = index_->d; @@ -84,9 +58,6 @@ bool Index::add_with_ids(idx_t n, const float *xdata, const long *xids) { bool Index::search(idx_t n, const float *data, idx_t k, float *distances, long *labels) const { try { - if(auto ivf_index = std::dynamic_pointer_cast(index_)) { - ivf_index->nprobe = Nprobe::GetInstance().GetNprobe(); - } index_->search(n, data, k, distances, labels); } catch (std::exception &e) { From 81e13fd1cbd134b40cd0f76461e43fd0ca00faab Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 3 Jul 2019 11:18:34 +0800 Subject: [PATCH 6/6] change CHANGELOG Former-commit-id: f1b0bebcca37b9fb33efc3faea4b9db66a9a4a7e --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index be89f31d11..bca5826ccb 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -62,6 +62,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-105 - Add MySQL - MS-130 - Add prometheus_test - MS-144 - Add nprobe config +- MS-147 - Enable IVF ## Task - MS-74 - Change README.md in cpp