diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 960b732d9d..845ec2ceda 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -16,6 +16,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-156 - Add unittest for merge result functions - MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl - MS-204 - Support multi db_path +- MS-206 - Support SQ8 index type ## New Feature - MS-195 - Add nlist and use_blas_threshold conf diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index d0029d5175..00e2e1e87d 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -189,4 +189,4 @@ install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 DESTINATION lib) #need to copy libmysqlpp.so -#add_subdirectory(sdk) +add_subdirectory(sdk) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index e6381dbbf9..628231db98 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -487,7 +487,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //step 6: update meta table_file.file_type_ = meta::TableFileSchema::INDEX; - table_file.size_ = index->Size(); + table_file.size_ = index->PhysicalSize(); auto to_remove = file; to_remove.file_type_ = meta::TableFileSchema::TO_DELETE; @@ -503,7 +503,9 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //index->Cache(); } catch (std::exception& ex) { - return Status::Error("Build index encounter exception", ex.what()); + std::string msg = "Build index encounter exception" + std::string(ex.what()); + ENGINE_LOG_ERROR << msg; + return Status::Error(msg); } return Status::OK(); diff --git a/cpp/src/db/EngineFactory.cpp b/cpp/src/db/EngineFactory.cpp index bacce70ce4..f45fb103b9 100644 --- a/cpp/src/db/EngineFactory.cpp +++ b/cpp/src/db/EngineFactory.cpp @@ -32,6 +32,12 @@ EngineFactory::Build(uint16_t dimension, break; } + case EngineType::FAISS_IVFSQ8: { + execution_engine_ptr = + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVFSQ8", "IDMap,Flat")); + break; + } + default: { ENGINE_LOG_ERROR << "Unsupported engine type"; return nullptr; diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/ExecutionEngine.h index f8c05f6f9d..4503e55650 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/ExecutionEngine.h @@ -18,6 +18,7 @@ enum class EngineType { INVALID = 0, FAISS_IDMAP = 1, FAISS_IVFFLAT, + FAISS_IVFSQ8, }; class ExecutionEngine { diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index a24c8de25d..26f144ff19 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -5,6 +5,7 @@ ******************************************************************************/ #include "FaissExecutionEngine.h" #include "Log.h" +#include "utils/CommonUtil.h" #include #include @@ -60,7 +61,7 @@ size_t FaissExecutionEngine::Dimension() const { } size_t FaissExecutionEngine::PhysicalSize() const { - return (size_t)(Count() * pIndex_->d)*sizeof(float); + return server::CommonUtil::GetFileSize(location_); } Status FaissExecutionEngine::Serialize() { diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index b356128898..f2513ac2c3 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -21,7 +21,7 @@ namespace { static constexpr int64_t NQ = 10; static constexpr int64_t TOP_K = 10; static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different - static constexpr int64_t ADD_VECTOR_LOOP = 5; + static constexpr int64_t ADD_VECTOR_LOOP = 10; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; @@ -97,7 +97,7 @@ namespace { TableSchema BuildTableSchema() { TableSchema tb_schema; tb_schema.table_name = TABLE_NAME; - tb_schema.index_type = IndexType::cpu_idmap; + tb_schema.index_type = IndexType::gpu_ivfsq8; tb_schema.dimension = TABLE_DIMENSION; tb_schema.store_raw_vector = true; diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index cb6d1f753e..ec14621981 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -18,6 +18,7 @@ enum class IndexType { invalid = 0, cpu_idmap, gpu_ivfflat, + gpu_ivfsq8, }; /** diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index e3a7286a7a..3f42b68531 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -30,6 +30,7 @@ namespace { {0, engine::EngineType::INVALID}, {1, engine::EngineType::FAISS_IDMAP}, {2, engine::EngineType::FAISS_IVFFLAT}, + {3, engine::EngineType::FAISS_IVFSQ8}, }; if(map_type.find(type) == map_type.end()) { @@ -44,6 +45,7 @@ namespace { {engine::EngineType::INVALID, 0}, {engine::EngineType::FAISS_IDMAP, 1}, {engine::EngineType::FAISS_IVFFLAT, 2}, + {engine::EngineType::FAISS_IVFSQ8, 3}, }; if(map_type.find(type) == map_type.end()) { diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index 00e3a8c94e..c2433b3231 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -140,6 +140,15 @@ bool CommonUtil::IsFileExist(const std::string &path) { return (access(path.c_str(), F_OK) == 0); } +uint64_t CommonUtil::GetFileSize(const std::string &path) { + struct stat fileInfo; + if (stat(path.c_str(), &fileInfo) < 0) { + return 0; + } else { + return (uint64_t)fileInfo.st_size; + } +} + std::string CommonUtil::GetExePath() { const size_t buf_len = 1024; char buf[buf_len]; diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index f1db3c059d..76d90c72ba 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -20,6 +20,7 @@ class CommonUtil { static bool GetSystemAvailableThreads(unsigned int &threadCnt); static bool IsFileExist(const std::string &path); + static uint64_t GetFileSize(const std::string &path); static bool IsDirectoryExist(const std::string &path); static ServerError CreateDirectory(const std::string &path); static ServerError DeleteDirectory(const std::string &path); diff --git a/cpp/src/wrapper/Operand.cpp b/cpp/src/wrapper/Operand.cpp index 0d526198ad..62a863ae0e 100644 --- a/cpp/src/wrapper/Operand.cpp +++ b/cpp/src/wrapper/Operand.cpp @@ -17,12 +17,14 @@ using std::string; enum IndexType { Invalid_Option = 0, IVF = 1, - IDMAP = 2 + IDMAP = 2, + IVFSQ8 = 3, }; IndexType resolveIndexType(const string &index_type) { if (index_type == "IVF") { return IndexType::IVF; } if (index_type == "IDMap") { return IndexType::IDMAP; } + if (index_type == "IVFSQ8") { return IndexType::IVFSQ8; } return IndexType::Invalid_Option; } @@ -30,9 +32,6 @@ IndexType resolveIndexType(const string &index_type) { string Operand::get_index_type(const int &nb) { if (!index_str.empty()) { return index_str; } - // TODO: support OPQ or ... - if (!preproc.empty()) { index_str += (preproc + ","); } - switch (resolveIndexType(index_type)) { case Invalid_Option: { // TODO: add exception @@ -47,17 +46,30 @@ string Operand::get_index_type(const int &nb) { index_str += (ncent != 0 ? index_type + std::to_string(ncent) : index_type + std::to_string(int(nb / 1000000.0 * nlist))); +// std::cout<<"nlist = "<