diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 960b732d9d..845ec2ceda 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -16,6 +16,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-156 - Add unittest for merge result functions - MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl - MS-204 - Support multi db_path +- MS-206 - Support SQ8 index type ## New Feature - MS-195 - Add nlist and use_blas_threshold conf diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index d0029d5175..00e2e1e87d 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -189,4 +189,4 @@ install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 DESTINATION lib) #need to copy libmysqlpp.so -#add_subdirectory(sdk) +add_subdirectory(sdk) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 8a8c2b117e..35a6f075f8 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -487,7 +487,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //step 6: update meta table_file.file_type_ = meta::TableFileSchema::INDEX; - table_file.size_ = index->Size(); + table_file.size_ = index->PhysicalSize(); auto to_remove = file; to_remove.file_type_ = meta::TableFileSchema::TO_DELETE; @@ -503,7 +503,9 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //index->Cache(); } catch (std::exception& ex) { - return Status::Error("Build index encounter exception", ex.what()); + std::string msg = "Build index encounter exception" + std::string(ex.what()); + ENGINE_LOG_ERROR << msg; + return Status::Error(msg); } return Status::OK(); diff --git a/cpp/src/db/EngineFactory.cpp b/cpp/src/db/EngineFactory.cpp index bacce70ce4..f45fb103b9 100644 --- a/cpp/src/db/EngineFactory.cpp +++ b/cpp/src/db/EngineFactory.cpp @@ -32,6 +32,12 @@ EngineFactory::Build(uint16_t dimension, break; } + case EngineType::FAISS_IVFSQ8: { + execution_engine_ptr = + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVFSQ8", "IDMap,Flat")); + break; + } + default: { ENGINE_LOG_ERROR << "Unsupported engine type"; return nullptr; diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/ExecutionEngine.h index f8c05f6f9d..4503e55650 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/ExecutionEngine.h @@ -18,6 +18,7 @@ enum class EngineType { INVALID = 0, FAISS_IDMAP = 1, FAISS_IVFFLAT, + FAISS_IVFSQ8, }; class ExecutionEngine { diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index a24c8de25d..26f144ff19 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -5,6 +5,7 @@ ******************************************************************************/ #include "FaissExecutionEngine.h" #include "Log.h" +#include "utils/CommonUtil.h" #include #include @@ -60,7 +61,7 @@ size_t FaissExecutionEngine::Dimension() const { } size_t FaissExecutionEngine::PhysicalSize() const { - return (size_t)(Count() * pIndex_->d)*sizeof(float); + return server::CommonUtil::GetFileSize(location_); } Status FaissExecutionEngine::Serialize() { diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index b356128898..f2513ac2c3 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -21,7 +21,7 @@ namespace { static constexpr int64_t NQ = 10; static constexpr int64_t TOP_K = 10; static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different - static constexpr int64_t ADD_VECTOR_LOOP = 5; + static constexpr int64_t ADD_VECTOR_LOOP = 10; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; @@ -97,7 +97,7 @@ namespace { TableSchema BuildTableSchema() { TableSchema tb_schema; tb_schema.table_name = TABLE_NAME; - tb_schema.index_type = IndexType::cpu_idmap; + tb_schema.index_type = IndexType::gpu_ivfsq8; tb_schema.dimension = TABLE_DIMENSION; tb_schema.store_raw_vector = true; diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index cb6d1f753e..ec14621981 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -18,6 +18,7 @@ enum class IndexType { invalid = 0, cpu_idmap, gpu_ivfflat, + gpu_ivfsq8, }; /** diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index e3a7286a7a..3f42b68531 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -30,6 +30,7 @@ namespace { {0, engine::EngineType::INVALID}, {1, engine::EngineType::FAISS_IDMAP}, {2, engine::EngineType::FAISS_IVFFLAT}, + {3, engine::EngineType::FAISS_IVFSQ8}, }; if(map_type.find(type) == map_type.end()) { @@ -44,6 +45,7 @@ namespace { {engine::EngineType::INVALID, 0}, {engine::EngineType::FAISS_IDMAP, 1}, {engine::EngineType::FAISS_IVFFLAT, 2}, + {engine::EngineType::FAISS_IVFSQ8, 3}, }; if(map_type.find(type) == map_type.end()) { diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index 00e3a8c94e..a376ee536f 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -32,21 +32,21 @@ namespace server { namespace fs = boost::filesystem; -bool CommonUtil::GetSystemMemInfo(unsigned long &totalMem, unsigned long &freeMem) { +bool CommonUtil::GetSystemMemInfo(unsigned long &total_mem, unsigned long &free_mem) { struct sysinfo info; int ret = sysinfo(&info); - totalMem = info.totalram; - freeMem = info.freeram; + total_mem = info.totalram; + free_mem = info.freeram; return ret == 0;//succeed 0, failed -1 } -bool CommonUtil::GetSystemAvailableThreads(unsigned int &threadCnt) { +bool CommonUtil::GetSystemAvailableThreads(unsigned int &thread_count) { //threadCnt = std::thread::hardware_concurrency(); - threadCnt = sysconf(_SC_NPROCESSORS_CONF); - threadCnt *= THREAD_MULTIPLY_CPU; - if (threadCnt == 0) - threadCnt = 8; + thread_count = sysconf(_SC_NPROCESSORS_CONF); + thread_count *= THREAD_MULTIPLY_CPU; + if (thread_count == 0) + thread_count = 8; return true; } @@ -66,9 +66,9 @@ ServerError CommonUtil::CreateDirectory(const std::string &path) { return SERVER_SUCCESS; } - struct stat directoryStat; - int statOK = stat(path.c_str(), &directoryStat); - if (statOK == 0) { + struct stat directory_stat; + int status = stat(path.c_str(), &directory_stat); + if (status == 0) { return SERVER_SUCCESS;//already exist } @@ -79,8 +79,8 @@ ServerError CommonUtil::CreateDirectory(const std::string &path) { return err; } - statOK = stat(path.c_str(), &directoryStat); - if (statOK == 0) { + status = stat(path.c_str(), &directory_stat); + if (status == 0) { return SERVER_SUCCESS;//already exist } @@ -94,29 +94,29 @@ ServerError CommonUtil::CreateDirectory(const std::string &path) { namespace { void RemoveDirectory(const std::string &path) { - DIR *pDir = NULL; + DIR *dir = nullptr; struct dirent *dmsg; - char szFileName[256]; - char szFolderName[256]; + char file_name[256]; + char folder_name[256]; - strcpy(szFolderName, path.c_str()); - strcat(szFolderName, "/%s"); - if ((pDir = opendir(path.c_str())) != NULL) { - while ((dmsg = readdir(pDir)) != NULL) { + strcpy(folder_name, path.c_str()); + strcat(folder_name, "/%s"); + if ((dir = opendir(path.c_str())) != nullptr) { + while ((dmsg = readdir(dir)) != nullptr) { if (strcmp(dmsg->d_name, ".") != 0 && strcmp(dmsg->d_name, "..") != 0) { - sprintf(szFileName, szFolderName, dmsg->d_name); - std::string tmp = szFileName; + sprintf(file_name, folder_name, dmsg->d_name); + std::string tmp = file_name; if (tmp.find(".") == std::string::npos) { - RemoveDirectory(szFileName); + RemoveDirectory(file_name); } - remove(szFileName); + remove(file_name); } } } - if (pDir != NULL) { - closedir(pDir); + if (dir != nullptr) { + closedir(dir); } remove(path.c_str()); } @@ -127,8 +127,8 @@ ServerError CommonUtil::DeleteDirectory(const std::string &path) { return SERVER_SUCCESS; } - struct stat directoryStat; - int statOK = stat(path.c_str(), &directoryStat); + struct stat directory_stat; + int statOK = stat(path.c_str(), &directory_stat); if (statOK != 0) return SERVER_SUCCESS; @@ -140,6 +140,15 @@ bool CommonUtil::IsFileExist(const std::string &path) { return (access(path.c_str(), F_OK) == 0); } +uint64_t CommonUtil::GetFileSize(const std::string &path) { + struct stat file_info; + if (stat(path.c_str(), &file_info) < 0) { + return 0; + } else { + return (uint64_t)file_info.st_size; + } +} + std::string CommonUtil::GetExePath() { const size_t buf_len = 1024; char buf[buf_len]; diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index f1db3c059d..095b237584 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -16,10 +16,11 @@ namespace server { class CommonUtil { public: - static bool GetSystemMemInfo(unsigned long &totalMem, unsigned long &freeMem); - static bool GetSystemAvailableThreads(unsigned int &threadCnt); + static bool GetSystemMemInfo(unsigned long &total_mem, unsigned long &free_mem); + static bool GetSystemAvailableThreads(unsigned int &thread_count); static bool IsFileExist(const std::string &path); + static uint64_t GetFileSize(const std::string &path); static bool IsDirectoryExist(const std::string &path); static ServerError CreateDirectory(const std::string &path); static ServerError DeleteDirectory(const std::string &path); diff --git a/cpp/src/wrapper/Operand.cpp b/cpp/src/wrapper/Operand.cpp index 0d526198ad..62a863ae0e 100644 --- a/cpp/src/wrapper/Operand.cpp +++ b/cpp/src/wrapper/Operand.cpp @@ -17,12 +17,14 @@ using std::string; enum IndexType { Invalid_Option = 0, IVF = 1, - IDMAP = 2 + IDMAP = 2, + IVFSQ8 = 3, }; IndexType resolveIndexType(const string &index_type) { if (index_type == "IVF") { return IndexType::IVF; } if (index_type == "IDMap") { return IndexType::IDMAP; } + if (index_type == "IVFSQ8") { return IndexType::IVFSQ8; } return IndexType::Invalid_Option; } @@ -30,9 +32,6 @@ IndexType resolveIndexType(const string &index_type) { string Operand::get_index_type(const int &nb) { if (!index_str.empty()) { return index_str; } - // TODO: support OPQ or ... - if (!preproc.empty()) { index_str += (preproc + ","); } - switch (resolveIndexType(index_type)) { case Invalid_Option: { // TODO: add exception @@ -47,17 +46,30 @@ string Operand::get_index_type(const int &nb) { index_str += (ncent != 0 ? index_type + std::to_string(ncent) : index_type + std::to_string(int(nb / 1000000.0 * nlist))); +// std::cout<<"nlist = "<