From 3fda33ac0bba66e9eac5b1162c860fcf0fbe299e Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Tue, 2 Jul 2019 19:36:06 +0800 Subject: [PATCH 1/3] alter nprobe Former-commit-id: 21cc3f6523580303bf702f7e616bcee7416a053f --- cpp/conf/server_config.yaml | 7 +++++-- cpp/src/wrapper/Index.cpp | 31 ++++++++++++++++++++++++++++++- cpp/src/wrapper/Operand.cpp | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 0053ba365a..359f8d86bf 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -5,7 +5,7 @@ server_config: mode: single # milvus deployment type: single, cluster db_config: - db_path: /opt/milvus # milvus data storage path + db_path: /tmp/milvus # milvus data storage path db_backend_url: http://127.0.0.1 # meta database uri index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB @@ -22,7 +22,10 @@ metric_config: license_config: # license configure - license_path: "/opt/milvus/system.license" # license file path + license_path: "/tmp/milvus/system.license" # license file path cache_config: # cache configure cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory + +engine_config: + nprobe: 3000 \ No newline at end of file diff --git a/cpp/src/wrapper/Index.cpp b/cpp/src/wrapper/Index.cpp index c306b65e1a..b051bfb07e 100644 --- a/cpp/src/wrapper/Index.cpp +++ b/cpp/src/wrapper/Index.cpp @@ -14,6 +14,8 @@ #include "Index.h" #include "faiss/index_io.h" #include "faiss/IndexIVF.h" +#include +#include "server/ServerConfig.h" namespace zilliz { namespace milvus { @@ -23,6 +25,32 @@ using std::string; using std::unordered_map; using std::vector; +class Nprobe { + public: + static Nprobe &GetInstance() { + static Nprobe instance; + return instance; + } + + void SelectNprobe() { + using namespace zilliz::milvus::server; + ServerConfig &config = ServerConfig::GetInstance(); + ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); + nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000); + } + + size_t GetNprobe() { + return nprobe_; + } + + private: + Nprobe() : nprobe_(1000) { SelectNprobe(); } + + private: + size_t nprobe_; +}; + + Index::Index(const std::shared_ptr &raw_index) { index_ = raw_index; dim = index_->d; @@ -57,7 +85,8 @@ bool Index::add_with_ids(idx_t n, const float *xdata, const long *xids) { bool Index::search(idx_t n, const float *data, idx_t k, float *distances, long *labels) const { try { if(auto ivf_index = std::dynamic_pointer_cast(index_)) { - ivf_index->nprobe = 100; + ivf_index->nprobe = Nprobe::GetInstance().GetNprobe(); + std::cout << "nprobe = " << ivf_index->nprobe << std::endl; } index_->search(n, data, k, distances, labels); } diff --git a/cpp/src/wrapper/Operand.cpp b/cpp/src/wrapper/Operand.cpp index 30e31067fd..25341676a6 100644 --- a/cpp/src/wrapper/Operand.cpp +++ b/cpp/src/wrapper/Operand.cpp @@ -39,7 +39,7 @@ string Operand::get_index_type(const int &nb) { } case IVF: { index_str += (ncent != 0 ? index_type + std::to_string(ncent) : - index_type + std::to_string(int(nb / 1000000.0 * 1638))); + index_type + std::to_string(int(nb / 1000000.0 * 16384))); break; } case IDMAP: { From 87af3d632de89fc976d2a6b7045b72ac1f1ca010 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Tue, 2 Jul 2019 20:13:10 +0800 Subject: [PATCH 2/3] add engine config Former-commit-id: 326501e3adc70b88fe3b8304ba39920fb11cf2d0 --- cpp/src/server/ServerConfig.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index f337275a46..412581bc1f 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -43,6 +43,9 @@ static const std::string CONFIG_METRIC_COLLECTOR = "collector"; static const std::string CONFIG_PROMETHEUS = "prometheus_config"; static const std::string CONFIG_METRIC_PROMETHEUS_PORT = "port"; +static const std::string CONFIG_ENGINE = "engine_config"; +static const std::string CONFIG_NPROBE = "nprobe"; + class ServerConfig { public: static ServerConfig &GetInstance(); From 380363f477b7ddb7f3181fa56369bb516ebbc04d Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Tue, 2 Jul 2019 20:24:41 +0800 Subject: [PATCH 3/3] update Former-commit-id: 99f1f53c6f2b1775f78acf0ba5560a112c68140b --- cpp/src/wrapper/Index.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/src/wrapper/Index.cpp b/cpp/src/wrapper/Index.cpp index b051bfb07e..57c462a201 100644 --- a/cpp/src/wrapper/Index.cpp +++ b/cpp/src/wrapper/Index.cpp @@ -14,7 +14,7 @@ #include "Index.h" #include "faiss/index_io.h" #include "faiss/IndexIVF.h" -#include +#include "faiss/IVFlib.h" #include "server/ServerConfig.h" namespace zilliz { @@ -86,7 +86,6 @@ bool Index::search(idx_t n, const float *data, idx_t k, float *distances, long * try { if(auto ivf_index = std::dynamic_pointer_cast(index_)) { ivf_index->nprobe = Nprobe::GetInstance().GetNprobe(); - std::cout << "nprobe = " << ivf_index->nprobe << std::endl; } index_->search(n, data, k, distances, labels); }