Merge remote-tracking branch 'upstream/branch-0.3.1' into integrate_knowhere-0.3.1

Former-commit-id: 21f32dea9d0f0a703248422758494bb87f621124

cpp/CHANGELOG.md

@@ -62,6 +62,8 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-105 - Add MySQL
 - MS-130 - Add prometheus_test
 - MS-143 - Intergrate Knowhere but not activate
+- MS-144 - Add nprobe config
+- MS-147 - Enable IVF
 
 ## Task
 - MS-74 - Change README.md in cpp

cpp/conf/server_config.template

@@ -30,4 +30,7 @@ license_config:         # license configure
   license_path: "@MILVUS_DB_PATH@/system.license" # license file path
 
 cache_config:           # cache configure
-  cpu_cache_capacity: 16   # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
+  cpu_cache_capacity: 16   # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
+
+engine_config:
+  nprobe: 10

cpp/src/db/EngineFactory.cpp

@@ -7,23 +7,39 @@
 #include "FaissExecutionEngine.h"
 #include "Log.h"
 
+
 namespace zilliz {
 namespace milvus {
 namespace engine {
 
 ExecutionEnginePtr
 EngineFactory::Build(uint16_t dimension,
-        const std::string& location,
-        EngineType type) {
-    switch(type) {
-        case EngineType::FAISS_IDMAP:
-            return ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat"));
-        case EngineType::FAISS_IVFFLAT:
-            return ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat"));
-        default:
-            ENGINE_LOG_ERROR << "Unsupportted engine type";
+                     const std::string &location,
+                     EngineType type) {
+
+    ExecutionEnginePtr execution_engine_ptr;
+
+    switch (type) {
+        case EngineType::FAISS_IDMAP: {
+            execution_engine_ptr =
+                ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat"));
+            break;
+        }
+
+        case EngineType::FAISS_IVFFLAT: {
+            execution_engine_ptr =
+                ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat"));
+            break;
+        }
+
+        default: {
+            ENGINE_LOG_ERROR << "Unsupported engine type";
             return nullptr;
+        }
     }
+
+    execution_engine_ptr->Init();
+    return execution_engine_ptr;
 }
 
 }

cpp/src/db/ExecutionEngine.h

@@ -50,6 +50,8 @@ public:
     virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string&) = 0;
 
     virtual Status Cache() = 0;
+
+    virtual Status Init() = 0;
 };
 
 using ExecutionEnginePtr = std::shared_ptr<ExecutionEngine>;

cpp/src/db/FaissExecutionEngine.cpp

@@ -13,6 +13,7 @@
 #include <wrapper/Index.h>
 #include <wrapper/IndexBuilder.h>
 #include <cache/CpuCacheMgr.h>
+#include "faiss/IndexIVF.h"
 #include "metrics/Metrics.h"
 
 
@@ -135,7 +136,16 @@ Status FaissExecutionEngine::Search(long n,
                                     float *distances,
                                     long *labels) const {
     auto start_time = METRICS_NOW_TIME;
-    pIndex_->search(n, data, k, distances, labels);
+
+    std::shared_ptr<faiss::IndexIVF> ivf_index = std::dynamic_pointer_cast<faiss::IndexIVF>(pIndex_);
+    if(ivf_index) {
+        ENGINE_LOG_DEBUG << "Index type: IVFFLAT nProbe: " << nprobe_;
+        ivf_index->nprobe = nprobe_;
+        ivf_index->search(n, data, k, distances, labels);
+    } else {
+        pIndex_->search(n, data, k, distances, labels);
+    }
+
     auto end_time = METRICS_NOW_TIME;
     auto total_time = METRICS_MICROSECONDS(start_time,end_time);
     server::Metrics::GetInstance().QueryIndexTypePerSecondSet(build_index_type_, double(n)/double(total_time));
@@ -149,6 +159,24 @@ Status FaissExecutionEngine::Cache() {
     return Status::OK();
 }
 
+Status FaissExecutionEngine::Init() {
+
+    if(build_index_type_ == "IVF") {
+
+        using namespace zilliz::milvus::server;
+        ServerConfig &config = ServerConfig::GetInstance();
+        ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE);
+        nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000);
+
+    } else if(build_index_type_ == "IDMap") {
+        ;
+    } else {
+        return Status::Error("Wrong index type: ", build_index_type_);
+    }
+
+    return Status::OK();
+}
+
 
 } // namespace engine
 } // namespace milvus

cpp/src/db/FaissExecutionEngine.h

@@ -6,14 +6,11 @@
 #pragma once
 
 #include "ExecutionEngine.h"
+#include "faiss/Index.h"
 
 #include <memory>
 #include <string>
 
-namespace faiss {
-    class Index;
-}
-
 namespace zilliz {
 namespace milvus {
 namespace engine {
@@ -58,12 +55,16 @@ public:
 
     Status Cache() override;
 
+    Status Init() override;
+
 protected:
     std::shared_ptr<faiss::Index> pIndex_;
     std::string location_;
 
     std::string build_index_type_;
     std::string raw_index_type_;
+
+    size_t nprobe_ = 0;
 };
 
 

cpp/src/main.cpp

@@ -26,7 +26,7 @@ using namespace zilliz::milvus;
 
 int
 main(int argc, char *argv[]) {
-    std::cout << std::endl << "Welcome to use Milvus by Zillz!" << std::endl;
+    std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl;
     std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl;
 
     signal(SIGINT, server::SignalUtil::HandleSignal);

cpp/src/server/ServerConfig.h

@@ -43,6 +43,9 @@ static const std::string CONFIG_METRIC_COLLECTOR = "collector";
 static const std::string CONFIG_PROMETHEUS = "prometheus_config";
 static const std::string CONFIG_METRIC_PROMETHEUS_PORT = "port";
 
+static const std::string CONFIG_ENGINE = "engine_config";
+static const std::string CONFIG_NPROBE = "nprobe";
+
 class ServerConfig {
  public:
     static ServerConfig &GetInstance();

cpp/src/wrapper/Index.cpp

@@ -14,6 +14,8 @@
 #include "Index.h"
 #include "faiss/index_io.h"
 #include "faiss/IndexIVF.h"
+#include "faiss/IVFlib.h"
+#include "server/ServerConfig.h"
 
 namespace zilliz {
 namespace milvus {
@@ -56,9 +58,6 @@ bool Index::add_with_ids(idx_t n, const float *xdata, const long *xids) {
 
 bool Index::search(idx_t n, const float *data, idx_t k, float *distances, long *labels) const {
     try {
-        if(auto ivf_index = std::dynamic_pointer_cast<faiss::IndexIVF>(index_)) {
-            ivf_index->nprobe = 100;
-        }
         index_->search(n, data, k, distances, labels);
     }
     catch (std::exception &e) {

cpp/src/wrapper/Operand.cpp

@@ -39,7 +39,7 @@ string Operand::get_index_type(const int &nb) {
         }
         case IVF: {
             index_str += (ncent != 0 ? index_type + std::to_string(ncent) :
-                          index_type + std::to_string(int(nb / 1000000.0 * 1638)));
+                          index_type + std::to_string(int(nb / 1000000.0 * 16384)));
             break;
         }
         case IDMAP: {

cpp/unittest/server/appendix/server_config.yaml

@@ -29,3 +29,6 @@ license_config:         # license configure
 
 cache_config:           # cache configure
   cpu_cache_capacity: 16   # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
+
+engine_config:
+  nprobe: 3000