MS-206 support SQ8 index type

Former-commit-id: 9788ff43d950e60a29b944d732648f73313c72d3

cpp/CHANGELOG.md

@@ -16,6 +16,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-156 - Add unittest for merge result functions
 - MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl
 - MS-204 - Support multi db_path
+- MS-206 - Support SQ8 index type
 
 ## New Feature
 - MS-195 - Add nlist and use_blas_threshold conf

cpp/src/CMakeLists.txt

@@ -189,4 +189,4 @@ install(FILES
         ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4
         DESTINATION lib) #need to copy libmysqlpp.so
 
-#add_subdirectory(sdk)
+add_subdirectory(sdk)

cpp/src/db/DBImpl.cpp

@@ -487,7 +487,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
 
         //step 6: update meta
         table_file.file_type_ = meta::TableFileSchema::INDEX;
-        table_file.size_ = index->Size();
+        table_file.size_ = index->PhysicalSize();
 
         auto to_remove = file;
         to_remove.file_type_ = meta::TableFileSchema::TO_DELETE;
@@ -503,7 +503,9 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
         //index->Cache();
 
     } catch (std::exception& ex) {
-        return Status::Error("Build index encounter exception", ex.what());
+        std::string msg = "Build index encounter exception" + std::string(ex.what());
+        ENGINE_LOG_ERROR << msg;
+        return Status::Error(msg);
     }
 
     return Status::OK();

cpp/src/db/EngineFactory.cpp

@@ -32,6 +32,12 @@ EngineFactory::Build(uint16_t dimension,
             break;
         }
 
+        case EngineType::FAISS_IVFSQ8: {
+            execution_engine_ptr =
+                    ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVFSQ8", "IDMap,Flat"));
+            break;
+        }
+
         default: {
             ENGINE_LOG_ERROR << "Unsupported engine type";
             return nullptr;

cpp/src/db/ExecutionEngine.h

@@ -18,6 +18,7 @@ enum class EngineType {
     INVALID = 0,
     FAISS_IDMAP = 1,
     FAISS_IVFFLAT,
+    FAISS_IVFSQ8,
 };
 
 class ExecutionEngine {

cpp/src/db/FaissExecutionEngine.cpp

@@ -5,6 +5,7 @@
  ******************************************************************************/
 #include "FaissExecutionEngine.h"
 #include "Log.h"
+#include "utils/CommonUtil.h"
 
 #include <faiss/AutoTune.h>
 #include <faiss/MetaIndexes.h>
@@ -60,7 +61,7 @@ size_t FaissExecutionEngine::Dimension() const {
 }
 
 size_t FaissExecutionEngine::PhysicalSize() const {
-    return (size_t)(Count() * pIndex_->d)*sizeof(float);
+    return server::CommonUtil::GetFileSize(location_);
 }
 
 Status FaissExecutionEngine::Serialize() {

cpp/src/sdk/examples/simple/src/ClientTest.cpp

@@ -21,7 +21,7 @@ namespace {
     static constexpr int64_t NQ = 10;
     static constexpr int64_t TOP_K = 10;
     static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different
-    static constexpr int64_t ADD_VECTOR_LOOP = 5;
+    static constexpr int64_t ADD_VECTOR_LOOP = 10;
 
 #define BLOCK_SPLITER std::cout << "===========================================" << std::endl;
 
@@ -97,7 +97,7 @@ namespace {
     TableSchema BuildTableSchema() {
         TableSchema tb_schema;
         tb_schema.table_name = TABLE_NAME;
-        tb_schema.index_type = IndexType::cpu_idmap;
+        tb_schema.index_type = IndexType::gpu_ivfsq8;
         tb_schema.dimension = TABLE_DIMENSION;
         tb_schema.store_raw_vector = true;
 

cpp/src/sdk/include/MilvusApi.h

@@ -18,6 +18,7 @@ enum class IndexType {
     invalid = 0,
     cpu_idmap,
     gpu_ivfflat,
+    gpu_ivfsq8,
 };
 
 /**

cpp/src/server/RequestTask.cpp

@@ -30,6 +30,7 @@ namespace {
                 {0, engine::EngineType::INVALID},
                 {1, engine::EngineType::FAISS_IDMAP},
                 {2, engine::EngineType::FAISS_IVFFLAT},
+                {3, engine::EngineType::FAISS_IVFSQ8},
         };
 
         if(map_type.find(type) == map_type.end()) {
@@ -44,6 +45,7 @@ namespace {
                 {engine::EngineType::INVALID, 0},
                 {engine::EngineType::FAISS_IDMAP, 1},
                 {engine::EngineType::FAISS_IVFFLAT, 2},
+                {engine::EngineType::FAISS_IVFSQ8, 3},
         };
 
         if(map_type.find(type) == map_type.end()) {

cpp/src/utils/CommonUtil.cpp

@@ -140,6 +140,15 @@ bool CommonUtil::IsFileExist(const std::string &path) {
     return (access(path.c_str(), F_OK) == 0);
 }
 
+uint64_t CommonUtil::GetFileSize(const std::string &path) {
+    struct stat fileInfo;
+    if (stat(path.c_str(), &fileInfo) < 0) {
+        return 0;
+    } else {
+        return (uint64_t)fileInfo.st_size;
+    }
+}
+
 std::string CommonUtil::GetExePath() {
     const size_t buf_len = 1024;
     char buf[buf_len];

cpp/src/utils/CommonUtil.h

@@ -20,6 +20,7 @@ class CommonUtil {
     static bool GetSystemAvailableThreads(unsigned int &threadCnt);
 
     static bool IsFileExist(const std::string &path);
+    static uint64_t GetFileSize(const std::string &path);
     static bool IsDirectoryExist(const std::string &path);
     static ServerError CreateDirectory(const std::string &path);
     static ServerError DeleteDirectory(const std::string &path);

cpp/src/wrapper/Operand.cpp

@@ -17,12 +17,14 @@ using std::string;
 enum IndexType {
     Invalid_Option = 0,
     IVF = 1,
-    IDMAP = 2
+    IDMAP = 2,
+    IVFSQ8 = 3,
 };
 
 IndexType resolveIndexType(const string &index_type) {
     if (index_type == "IVF") { return IndexType::IVF; }
     if (index_type == "IDMap") { return IndexType::IDMAP; }
+    if (index_type == "IVFSQ8") { return IndexType::IVFSQ8; }
     return IndexType::Invalid_Option;
 }
 
@@ -30,9 +32,6 @@ IndexType resolveIndexType(const string &index_type) {
 string Operand::get_index_type(const int &nb) {
     if (!index_str.empty()) { return index_str; }
 
-    // TODO: support OPQ or ...
-    if (!preproc.empty()) { index_str += (preproc + ","); }
-
     switch (resolveIndexType(index_type)) {
         case Invalid_Option: {
             // TODO: add exception
@@ -47,17 +46,30 @@ string Operand::get_index_type(const int &nb) {
 
             index_str += (ncent != 0 ? index_type + std::to_string(ncent) :
                           index_type + std::to_string(int(nb / 1000000.0 * nlist)));
+//            std::cout<<"nlist = "<<nlist<<std::endl;
+            if (!postproc.empty()) { index_str += ("," + postproc); }
+            break;
+        }
+        case IVFSQ8: {
+
+            using namespace zilliz::milvus::server;
+            ServerConfig &config = ServerConfig::GetInstance();
+            ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE);
+            size_t nlist = engine_config.GetInt32Value(CONFIG_NLIST, 16384);
+
+            index_str += (ncent != 0 ? "IVF" + std::to_string(ncent) :
+                          "IVF" + std::to_string(int(nb / 1000000.0 * nlist)));
+            index_str += ",SQ8";
 //            std::cout<<"nlist = "<<nlist<<std::endl;
             break;
         }
         case IDMAP: {
             index_str += index_type;
+            if (!postproc.empty()) { index_str += ("," + postproc); }
             break;
         }
     }
 
-    // TODO: support PQ or ...
-    if (!postproc.empty()) { index_str += ("," + postproc); }
     return index_str;
 }