From b285c64e96928e48d36a6cd9760dcfbb9e298ed7 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 22 Jul 2019 10:44:24 +0800 Subject: [PATCH 01/24] Check machine hardware during initialize Former-commit-id: fe3efe3b519235d9579e0c311a7da3f7a299154f --- cpp/CHANGELOG.md | 1 + cpp/conf/server_config.template | 2 +- cpp/src/CMakeLists.txt | 1 + cpp/src/server/RequestTask.cpp | 20 +++---- cpp/src/server/Server.cpp | 4 ++ cpp/src/server/ServerConfig.cpp | 64 +++++++++++++++++++++++ cpp/src/server/ServerConfig.h | 2 + cpp/src/utils/ValidationUtil.cpp | 38 ++++++++++++-- cpp/src/utils/ValidationUtil.h | 21 +++++--- cpp/src/wrapper/IndexBuilder.cpp | 2 +- cpp/unittest/CMakeLists.txt | 1 + cpp/unittest/db/CMakeLists.txt | 4 ++ cpp/unittest/faiss_wrapper/CMakeLists.txt | 4 ++ cpp/unittest/metrics/CMakeLists.txt | 4 ++ cpp/unittest/server/CMakeLists.txt | 1 + cpp/unittest/storage/CMakeLists.txt | 7 ++- cpp/unittest/utils/ValidationUtilTest.cpp | 30 +++++------ 17 files changed, 168 insertions(+), 38 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 56c5c5faa8..6e1fff4a2d 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -35,6 +35,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-248 - Support AddVector/SearchVector profiling - MS-256 - Add more cache config - MS-260 - Refine log +- MS-249 - Check machine hardware during initialize ## New Feature - MS-180 - Add new mem manager diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index b079d603d2..215ca45027 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -17,7 +17,7 @@ db_config: archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. - # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory + # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 64edf5b91e..5784734dd6 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -88,6 +88,7 @@ set(third_party_libs mysqlpp ${PROFILER_LIB} ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so + cudart ) if (MEGASEARCH_WITH_ARROW STREQUAL "ON") diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index d6c1a2fb9c..0ab6a6a5e4 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -162,17 +162,17 @@ ServerError CreateTableTask::OnExecute() { try { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(schema_.table_name); + res = ValidationUtil::ValidateTableName(schema_.table_name); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + schema_.table_name); } - res = ValidateTableDimension(schema_.dimension); + res = ValidationUtil::ValidateTableDimension(schema_.dimension); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table dimension: " + std::to_string(schema_.dimension)); } - res = ValidateTableIndexType(schema_.index_type); + res = ValidationUtil::ValidateTableIndexType(schema_.index_type); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid index type: " + std::to_string(schema_.index_type)); } @@ -217,7 +217,7 @@ ServerError DescribeTableTask::OnExecute() { try { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -260,7 +260,7 @@ ServerError BuildIndexTask::OnExecute() { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -303,7 +303,7 @@ ServerError HasTableTask::OnExecute() { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -339,7 +339,7 @@ ServerError DeleteTableTask::OnExecute() { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -420,7 +420,7 @@ ServerError AddVectorTask::OnExecute() { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -508,7 +508,7 @@ ServerError SearchVectorTaskBase::OnExecute() { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -720,7 +720,7 @@ ServerError GetTableRowCountTask::OnExecute() { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index a79a8c7357..aeccc1e5be 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -226,6 +226,10 @@ Server::Stop() { ServerError Server::LoadConfig() { ServerConfig::GetInstance().LoadConfigFile(config_filename_); + ServerError err = ServerConfig::GetInstance().ValidateConfig(); + if(err != SERVER_SUCCESS){ + exit(0); + } return SERVER_SUCCESS; } diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 736a249e25..bb1879e0b4 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -12,11 +12,16 @@ #include #include "config/IConfigMgr.h" +#include "utils/CommonUtil.h" +#include "utils/ValidationUtil.h" namespace zilliz { namespace milvus { namespace server { +constexpr uint64_t MB = 1024*1024; +constexpr uint64_t GB = MB*1024; + ServerConfig& ServerConfig::GetInstance() { static ServerConfig config; @@ -53,6 +58,65 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) { return SERVER_SUCCESS; } +ServerError ServerConfig::ValidateConfig() const { + //server config validation + ConfigNode server_config = GetConfig(CONFIG_SERVER); + uint32_t gpu_index = (uint32_t)server_config.GetInt32Value(CONFIG_GPU_INDEX); + if(ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) { + std::cout << "Error: invalid gpu_index " << std::to_string(gpu_index) << std::endl; + return SERVER_INVALID_ARGUMENT; + } + + //db config validation + unsigned long total_mem = 0, free_mem = 0; + CommonUtil::GetSystemMemInfo(total_mem, free_mem); + + ConfigNode db_config = GetConfig(CONFIG_DB); + uint64_t insert_buffer_size = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE); + insert_buffer_size *= GB; + if(insert_buffer_size >= total_mem) { + std::cout << "Error: insert_buffer_size execeed system memory" << std::endl; + return SERVER_INVALID_ARGUMENT; + } + + uint64_t index_building_threshold = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INDEX_TRIGGER_SIZE); + index_building_threshold *= MB; + + size_t gpu_mem = 0; + ValidationUtil::GetGpuMemory(gpu_index, gpu_mem); + if(index_building_threshold >= gpu_mem/3) { + std::cout << "Warnning: index_building_threshold is greater than 1/3 of gpu memory, " + << "some index type(such as IVFLAT) may cause cuda::bad_alloc() error" << std::endl; + } else if(index_building_threshold >= gpu_mem) { + std::cout << "Error: index_building_threshold execeed gpu memory" << std::endl; + return SERVER_INVALID_ARGUMENT; + } + + //cache config validation + ConfigNode cache_config = GetConfig(CONFIG_CACHE); + uint64_t cache_cap = (uint64_t)cache_config.GetInt64Value(CONFIG_CPU_CACHE_CAPACITY); + cache_cap *= GB; + if(cache_cap >= total_mem) { + std::cout << "Error: cpu_cache_capacity execeed system memory" << std::endl; + return SERVER_INVALID_ARGUMENT; + } if(cache_cap > (double)total_mem*0.9) { + std::cout << "Warnning: cpu_cache_capacity value is too aggressive" << std::endl; + } + + if(insert_buffer_size + cache_cap >= total_mem) if(cache_cap >= total_mem) { + std::cout << "Error: sum of cpu_cache_capacity and insert_buffer_size execeed system memory" << std::endl; + return SERVER_INVALID_ARGUMENT; + } + + double free_percent = cache_config.GetDoubleValue(server::CACHE_FREE_PERCENT); + if(free_percent < std::numeric_limits::epsilon() || free_percent > 1.0) { + std::cout << "Error: invalid cache_free_percent " << std::to_string(free_percent) << std::endl; + return SERVER_INVALID_ARGUMENT; + } + + return SERVER_SUCCESS; +} + void ServerConfig::PrintAll() const { if(const IConfigMgr* mgr = IConfigMgr::GetInstance()) { diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index 12d1e8b889..4a1b089810 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -19,6 +19,7 @@ static const std::string CONFIG_SERVER_ADDRESS = "address"; static const std::string CONFIG_SERVER_PORT = "port"; static const std::string CONFIG_SERVER_PROTOCOL = "transfer_protocol"; static const std::string CONFIG_CLUSTER_MODE = "mode"; +static const std::string CONFIG_GPU_INDEX = "gpu_index"; static const std::string CONFIG_DB = "db_config"; static const std::string CONFIG_DB_URL = "db_backend_url"; @@ -57,6 +58,7 @@ class ServerConfig { static ServerConfig &GetInstance(); ServerError LoadConfigFile(const std::string& config_filename); + ServerError ValidateConfig() const; void PrintAll() const; ConfigNode GetConfig(const std::string& name) const; diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index 89b5573999..bfdf35cee2 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -1,7 +1,8 @@ -#include +#include "db/ExecutionEngine.h" #include "ValidationUtil.h" #include "Log.h" +#include namespace zilliz { namespace milvus { @@ -11,7 +12,7 @@ constexpr size_t table_name_size_limit = 255; constexpr int64_t table_dimension_limit = 16384; ServerError -ValidateTableName(const std::string &table_name) { +ValidationUtil::ValidateTableName(const std::string &table_name) { // Table name shouldn't be empty. if (table_name.empty()) { @@ -45,7 +46,7 @@ ValidateTableName(const std::string &table_name) { } ServerError -ValidateTableDimension(int64_t dimension) { +ValidationUtil::ValidateTableDimension(int64_t dimension) { if (dimension <= 0 || dimension > table_dimension_limit) { SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit; return SERVER_INVALID_VECTOR_DIMENSION; @@ -55,7 +56,7 @@ ValidateTableDimension(int64_t dimension) { } ServerError -ValidateTableIndexType(int32_t index_type) { +ValidationUtil::ValidateTableIndexType(int32_t index_type) { int engine_type = (int)engine::EngineType(index_type); if(engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) { return SERVER_INVALID_INDEX_TYPE; @@ -64,6 +65,35 @@ ValidateTableIndexType(int32_t index_type) { return SERVER_SUCCESS; } +ServerError +ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { + int num_devices = 0; + auto cuda_err = cudaGetDeviceCount(&num_devices); + if (cuda_err) { + SERVER_LOG_ERROR << "Failed to count video card: " << std::to_string(cuda_err); + return SERVER_UNEXPECTED_ERROR; + } + + if(gpu_index >= num_devices) { + return SERVER_INVALID_ARGUMENT; + } + + return SERVER_SUCCESS; +} + +ServerError +ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { + cudaDeviceProp deviceProp; + auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); + if (cuda_err) { + SERVER_LOG_ERROR << "Failed to get video card properties: " << std::to_string(cuda_err); + return SERVER_UNEXPECTED_ERROR; + } + + memory = deviceProp.totalGlobalMem; + return SERVER_SUCCESS; +} + } } } \ No newline at end of file diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index 608ac22682..1f90fac273 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -6,14 +6,23 @@ namespace zilliz { namespace milvus { namespace server { -ServerError -ValidateTableName(const std::string& table_name); +class ValidationUtil { +public: + static ServerError + ValidateTableName(const std::string &table_name); -ServerError -ValidateTableDimension(int64_t dimension); + static ServerError + ValidateTableDimension(int64_t dimension); -ServerError -ValidateTableIndexType(int32_t index_type); + static ServerError + ValidateTableIndexType(int32_t index_type); + + static ServerError + ValidateGpuIndex(uint32_t gpu_index); + + static ServerError + GetGpuMemory(uint32_t gpu_index, size_t &memory); +}; } } diff --git a/cpp/src/wrapper/IndexBuilder.cpp b/cpp/src/wrapper/IndexBuilder.cpp index 41859907a7..62781751e2 100644 --- a/cpp/src/wrapper/IndexBuilder.cpp +++ b/cpp/src/wrapper/IndexBuilder.cpp @@ -37,7 +37,7 @@ class GpuResources { using namespace zilliz::milvus::server; ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_SERVER); - gpu_num = server_config.GetInt32Value("gpu_index", 0); + gpu_num = server_config.GetInt32Value(server::CONFIG_GPU_INDEX, 0); } int32_t GetGpu() { diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 6a6f94a632..f4a670acf3 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -35,6 +35,7 @@ set(unittest_libs dl z ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so + cudart ) add_subdirectory(server) diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 736219952c..a7bed578dd 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -9,6 +9,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) aux_source_directory(./ test_srcs) +set(util_files + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) + aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) @@ -29,6 +32,7 @@ set(db_test_src ${db_srcs} ${db_scheduler_srcs} ${wrapper_src} + ${util_files} ${require_files} ${test_srcs}) diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index c439250544..c906c8ec57 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -6,6 +6,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) +set(util_files + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) + # Make sure that your call to link_directories takes place before your call to the relevant add_executable. include_directories(/usr/local/cuda/include) link_directories("/usr/local/cuda/lib64") @@ -14,6 +17,7 @@ set(wrapper_test_src ${unittest_srcs} ${wrapper_src} ${config_files} + ${util_files} ${require_files} wrapper_test.cpp ) diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 418544d0ca..661201fbc0 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -17,6 +17,9 @@ aux_source_directory(../../src/wrapper wrapper_src) aux_source_directory(../../src/metrics metrics_src) aux_source_directory(./ test_srcs) +set(util_files + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) + aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) @@ -43,6 +46,7 @@ set(count_test_src ${wrapper_src} ${metrics_src} ${test_srcs} + ${util_files} ) diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 40f5b10128..0eb6c7cd10 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -19,6 +19,7 @@ set(utils_srcs ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp ${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp ) cuda_add_executable(server_test diff --git a/cpp/unittest/storage/CMakeLists.txt b/cpp/unittest/storage/CMakeLists.txt index d4deaefab8..7529028e3c 100644 --- a/cpp/unittest/storage/CMakeLists.txt +++ b/cpp/unittest/storage/CMakeLists.txt @@ -5,6 +5,9 @@ #------------------------------------------------------------------------------- aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 s3_client_src) +set(util_files + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) + # Make sure that your call to link_directories takes place before your call to the relevant add_executable. include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") @@ -19,7 +22,9 @@ set(s3_client_test_src add_executable(s3_test ${s3_client_test_src} - ${config_files}) + ${config_files} + ${util_files} + ) set(s3_client_libs stdc++ diff --git a/cpp/unittest/utils/ValidationUtilTest.cpp b/cpp/unittest/utils/ValidationUtilTest.cpp index 38fc63a10d..871172e6c9 100644 --- a/cpp/unittest/utils/ValidationUtilTest.cpp +++ b/cpp/unittest/utils/ValidationUtilTest.cpp @@ -16,48 +16,48 @@ using namespace zilliz::milvus::server; TEST(ValidationUtilTest, TableNameTest) { std::string table_name = "Normal123_"; - ServerError res = ValidateTableName(table_name); + ServerError res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_SUCCESS); table_name = "12sds"; - res = ValidateTableName(table_name); + res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = ""; - res = ValidateTableName(table_name); + res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = "_asdasd"; - res = ValidateTableName(table_name); + res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_SUCCESS); table_name = "!@#!@"; - res = ValidateTableName(table_name); + res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = "中文"; - res = ValidateTableName(table_name); + res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = std::string('a', 32768); - res = ValidateTableName(table_name); + res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); } TEST(ValidationUtilTest, TableDimensionTest) { - ASSERT_EQ(ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ValidateTableDimension(16384), SERVER_SUCCESS); - ASSERT_EQ(ValidateTableDimension(1), SERVER_SUCCESS); + ASSERT_EQ(ValidationUtil::ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ValidationUtil::ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ValidationUtil::ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ValidationUtil::ValidateTableDimension(16384), SERVER_SUCCESS); + ASSERT_EQ(ValidationUtil::ValidateTableDimension(1), SERVER_SUCCESS); } TEST(ValidationUtilTest, TableIndexTypeTest) { - ASSERT_EQ(ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE); for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) { - ASSERT_EQ(ValidateTableIndexType(i), SERVER_SUCCESS); + ASSERT_EQ(ValidationUtil::ValidateTableIndexType(i), SERVER_SUCCESS); } - ASSERT_EQ(ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE); } From eb1d1e393f9cb56205ac2ab86a8250bf94b1348b Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 22 Jul 2019 11:15:11 +0800 Subject: [PATCH 02/24] MS-249 Check machine hardware during initialize Former-commit-id: e513499fb015a6867a762a10b629dd774182f21a --- cpp/src/server/ServerConfig.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index bb1879e0b4..4a5d0d0931 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -84,12 +84,12 @@ ServerError ServerConfig::ValidateConfig() const { size_t gpu_mem = 0; ValidationUtil::GetGpuMemory(gpu_index, gpu_mem); - if(index_building_threshold >= gpu_mem/3) { - std::cout << "Warnning: index_building_threshold is greater than 1/3 of gpu memory, " - << "some index type(such as IVFLAT) may cause cuda::bad_alloc() error" << std::endl; - } else if(index_building_threshold >= gpu_mem) { + if(index_building_threshold >= gpu_mem) { std::cout << "Error: index_building_threshold execeed gpu memory" << std::endl; return SERVER_INVALID_ARGUMENT; + } else if(index_building_threshold >= gpu_mem/3) { + std::cout << "Warnning: index_building_threshold is greater than 1/3 of gpu memory, " + << "some index type(such as IVFLAT) may cause cuda::bad_alloc() error" << std::endl; } //cache config validation @@ -103,7 +103,7 @@ ServerError ServerConfig::ValidateConfig() const { std::cout << "Warnning: cpu_cache_capacity value is too aggressive" << std::endl; } - if(insert_buffer_size + cache_cap >= total_mem) if(cache_cap >= total_mem) { + if(insert_buffer_size + cache_cap >= total_mem) { std::cout << "Error: sum of cpu_cache_capacity and insert_buffer_size execeed system memory" << std::endl; return SERVER_INVALID_ARGUMENT; } From 0ed96e5f2f363920bbb761080a5a5d05bd486f76 Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 22 Jul 2019 11:16:14 +0800 Subject: [PATCH 03/24] fix Former-commit-id: 162878ec45e3cd98555f08a1d3cf7fc343aa477a --- cpp/unittest/db/CMakeLists.txt | 4 ++++ cpp/unittest/faiss_wrapper/CMakeLists.txt | 4 ++++ cpp/unittest/metrics/CMakeLists.txt | 4 ++++ cpp/unittest/server/CMakeLists.txt | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 0e3531cd33..0fbaf85073 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -47,6 +47,10 @@ set(db_libs if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) +else() + set(db_libs ${db_libs} + lapack + openblas) endif() target_link_libraries(db_test ${db_libs} ${unittest_libs}) diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index f7976fb604..7b12309b2d 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -36,6 +36,10 @@ set(wrapper_libs ) if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") set(wrapper_libs ${wrapper_libs} ${MKL_LIBS} ${MKL_LIBS}) +else() + set(wrapper_libs ${wrapper_libs} + lapack + openblas) endif() target_link_libraries(wrapper_test ${wrapper_libs} ${unittest_libs}) diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index d4ae934d6f..60795f9ce5 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -65,6 +65,10 @@ target_link_libraries(metrics_test ) if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS}) +else() + target_link_libraries(metrics_test + lapack + openblas) endif() install(TARGETS metrics_test DESTINATION bin) \ No newline at end of file diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 7a20e6287b..9b4f7570c9 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -49,6 +49,10 @@ set(require_libs if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS}) +else() + set(require_libs ${require_libs} + lapack + openblas) endif() target_link_libraries(server_test From 2671c711f6ada431ca829b2a288567e62b89874e Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 22 Jul 2019 14:00:09 +0800 Subject: [PATCH 04/24] update test namespace Former-commit-id: 6adcb38e41730728e6df89be6da343c82e6f6f31 --- ci/jenkinsfile/deploy2dev.groovy | 8 -------- ci/jenkinsfile/dev_test.groovy | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index 52cbae2bfe..e980699f1f 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -8,14 +8,6 @@ try { sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.3.1 ." } } - /* - timeout(time: 2, unit: 'MINUTES') { - waitUntil { - def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true - return !result - } - } - */ } catch (exc) { echo 'Helm running failed!' sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index fb5b4749cc..1b6ef4de13 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -3,7 +3,7 @@ timeout(time: 20, unit: 'MINUTES') { dir ("${PROJECT_NAME}_test") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" + sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" } // mysql database backend test @@ -20,7 +20,7 @@ timeout(time: 20, unit: 'MINUTES') { } } dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" + sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local" } } catch (exc) { echo 'Milvus Test Failed !' From f7742f7281c937373946505c840de9ba0705a7c9 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 22 Jul 2019 14:25:33 +0800 Subject: [PATCH 05/24] MS-249 Check machine hardware during initialize Former-commit-id: 874dcaa8f6000158409de2865ca28050e4ceeac3 --- cpp/conf/server_config.template | 2 +- cpp/src/cache/Cache.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 215ca45027..2942ffa179 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -34,7 +34,7 @@ license_config: # license configure cache_config: # cache configure cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory - cache_free_percent: 0.85 # how much memory should be free when cache is full, range: greater than zero ~ 1.0 + cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 insert_cache_immediately: false # insert data will be load into cache immediately for hot query engine_config: diff --git a/cpp/src/cache/Cache.cpp b/cpp/src/cache/Cache.cpp index 0cc804ac5f..a1f2520302 100644 --- a/cpp/src/cache/Cache.cpp +++ b/cpp/src/cache/Cache.cpp @@ -163,6 +163,9 @@ void Cache::free_memory() { int64_t threshhold = capacity_ * freemem_percent_; int64_t delta_size = usage_ - threshhold; + if(delta_size <= 0) { + delta_size = 1;//ensure at least one item erased + } std::set key_array; int64_t released_size = 0; From add911ae22aed014a7f0785106fa1222a538f55c Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 22 Jul 2019 15:33:36 +0800 Subject: [PATCH 06/24] MS-249 Check machine hardware during initialize Former-commit-id: 9f42d9c41dcfac9bc1716737368826bfcd777251 --- cpp/src/server/ServerConfig.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 4a5d0d0931..71d32452ac 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -61,7 +61,7 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) { ServerError ServerConfig::ValidateConfig() const { //server config validation ConfigNode server_config = GetConfig(CONFIG_SERVER); - uint32_t gpu_index = (uint32_t)server_config.GetInt32Value(CONFIG_GPU_INDEX); + uint32_t gpu_index = (uint32_t)server_config.GetInt32Value(CONFIG_GPU_INDEX, 0); if(ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) { std::cout << "Error: invalid gpu_index " << std::to_string(gpu_index) << std::endl; return SERVER_INVALID_ARGUMENT; @@ -72,14 +72,14 @@ ServerError ServerConfig::ValidateConfig() const { CommonUtil::GetSystemMemInfo(total_mem, free_mem); ConfigNode db_config = GetConfig(CONFIG_DB); - uint64_t insert_buffer_size = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE); + uint64_t insert_buffer_size = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4); insert_buffer_size *= GB; if(insert_buffer_size >= total_mem) { std::cout << "Error: insert_buffer_size execeed system memory" << std::endl; return SERVER_INVALID_ARGUMENT; } - uint64_t index_building_threshold = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INDEX_TRIGGER_SIZE); + uint64_t index_building_threshold = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INDEX_TRIGGER_SIZE, 1024); index_building_threshold *= MB; size_t gpu_mem = 0; @@ -94,7 +94,7 @@ ServerError ServerConfig::ValidateConfig() const { //cache config validation ConfigNode cache_config = GetConfig(CONFIG_CACHE); - uint64_t cache_cap = (uint64_t)cache_config.GetInt64Value(CONFIG_CPU_CACHE_CAPACITY); + uint64_t cache_cap = (uint64_t)cache_config.GetInt64Value(CONFIG_CPU_CACHE_CAPACITY, 16); cache_cap *= GB; if(cache_cap >= total_mem) { std::cout << "Error: cpu_cache_capacity execeed system memory" << std::endl; @@ -108,7 +108,7 @@ ServerError ServerConfig::ValidateConfig() const { return SERVER_INVALID_ARGUMENT; } - double free_percent = cache_config.GetDoubleValue(server::CACHE_FREE_PERCENT); + double free_percent = cache_config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85); if(free_percent < std::numeric_limits::epsilon() || free_percent > 1.0) { std::cout << "Error: invalid cache_free_percent " << std::to_string(free_percent) << std::endl; return SERVER_INVALID_ARGUMENT; From f10d8e1b664262c36c2cea6c90434f1b82821bf1 Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 22 Jul 2019 17:02:03 +0800 Subject: [PATCH 07/24] change BUILD_FAISS_WITH_MKL to default on Former-commit-id: 2efee7ecceb466100ea0d875aeb1aeb3dbc02737 --- cpp/build.sh | 2 +- cpp/cmake/ThirdPartyPackages.cmake | 2 +- cpp/src/CMakeLists.txt | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/build.sh b/cpp/build.sh index edfe9305be..ef66a0c375 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -8,7 +8,7 @@ MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/opt/milvus" PROFILING="OFF" -BUILD_FAISS_WITH_MKL="OFF" +BUILD_FAISS_WITH_MKL="ON" while getopts "p:d:t:uhlrcgm" arg do diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index fed5dece88..837d6bd470 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -700,7 +700,7 @@ endmacro() # FAISS if(NOT DEFINED BUILD_FAISS_WITH_MKL) - set(BUILD_FAISS_WITH_MKL OFF) + set(BUILD_FAISS_WITH_MKL ON) endif() if(EXISTS "/proc/cpuinfo") diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 80603cf673..088a461a9b 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -153,6 +153,7 @@ if (ENABLE_LICENSE STREQUAL "ON") endif () set(metrics_lib + easyloggingpp prometheus-cpp-push prometheus-cpp-pull prometheus-cpp-core From 5f86066e8d51a5eaa18fb3eb15361e1fe7ed15ec Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 22 Jul 2019 17:49:11 +0800 Subject: [PATCH 08/24] MS-246 refine log Former-commit-id: d8c30756db5666f964fd4fcf63fdc4a7ab08a4ea --- cpp/src/server/RequestTask.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 0ab6a6a5e4..d9e50a82bd 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -504,7 +504,9 @@ SearchVectorTaskBase::SearchVectorTaskBase(const std::string &table_name, ServerError SearchVectorTaskBase::OnExecute() { try { - TimeRecorder rc("SearchVectorTask"); + std::string title = "SearchVectorTask(n=" + std::to_string(record_array_.size()) + + " k=" + std::to_string(top_k_) + ")"; + TimeRecorder rc(title); //step 1: check arguments ServerError res = SERVER_SUCCESS; @@ -596,7 +598,7 @@ ServerError SearchVectorTaskBase::OnExecute() { //step 6: print time cost percent double total_cost = span_check + span_prepare + span_search + span_result; - SERVER_LOG_DEBUG << "SearchVectorTask: " << "check validation(" << (span_check/total_cost)*100.0 << "%)" + SERVER_LOG_DEBUG << title << ": check validation(" << (span_check/total_cost)*100.0 << "%)" << " prepare data(" << (span_prepare/total_cost)*100.0 << "%)" << " search(" << (span_search/total_cost)*100.0 << "%)" << " construct result(" << (span_result/total_cost)*100.0 << "%)"; From 397d9516f301c9e2ebc8f23bd47ed9b3d62413d5 Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 22 Jul 2019 18:24:26 +0800 Subject: [PATCH 09/24] change BUILD_FAISS_WITH_MKL to default off Former-commit-id: 2d61d5e7e25732ed8632d295992ac16378181c37 --- cpp/build.sh | 2 +- cpp/cmake/ThirdPartyPackages.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/build.sh b/cpp/build.sh index ef66a0c375..edfe9305be 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -8,7 +8,7 @@ MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/opt/milvus" PROFILING="OFF" -BUILD_FAISS_WITH_MKL="ON" +BUILD_FAISS_WITH_MKL="OFF" while getopts "p:d:t:uhlrcgm" arg do diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 837d6bd470..fed5dece88 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -700,7 +700,7 @@ endmacro() # FAISS if(NOT DEFINED BUILD_FAISS_WITH_MKL) - set(BUILD_FAISS_WITH_MKL ON) + set(BUILD_FAISS_WITH_MKL OFF) endif() if(EXISTS "/proc/cpuinfo") From cdb031e62bd9781b3c051f52e46bda8d27706487 Mon Sep 17 00:00:00 2001 From: "peng.xu" Date: Mon, 22 Jul 2019 20:03:59 +0800 Subject: [PATCH 10/24] Revert "Merge branch 'branch-0.3.1' into 'branch-0.3.1'" This reverts merge request !264 Former-commit-id: 501e92946ab87fb9a21c55545f84ab8da499e1aa --- cpp/build.sh | 2 +- cpp/cmake/ThirdPartyPackages.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/build.sh b/cpp/build.sh index edfe9305be..ef66a0c375 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -8,7 +8,7 @@ MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/opt/milvus" PROFILING="OFF" -BUILD_FAISS_WITH_MKL="OFF" +BUILD_FAISS_WITH_MKL="ON" while getopts "p:d:t:uhlrcgm" arg do diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index fed5dece88..837d6bd470 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -700,7 +700,7 @@ endmacro() # FAISS if(NOT DEFINED BUILD_FAISS_WITH_MKL) - set(BUILD_FAISS_WITH_MKL OFF) + set(BUILD_FAISS_WITH_MKL ON) endif() if(EXISTS "/proc/cpuinfo") From 28e84f384d743e3519e86d73ee050e97c90cce24 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 23 Jul 2019 10:41:28 +0800 Subject: [PATCH 11/24] fix metrics Former-commit-id: 2a7f67913dfdb615e09d7316053c59944132a8fe --- cpp/src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 088a461a9b..882e098258 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -154,6 +154,7 @@ endif () set(metrics_lib easyloggingpp + yaml-cpp prometheus-cpp-push prometheus-cpp-pull prometheus-cpp-core From 57fd142712ef8fdbed430256c2644bc8cbe32d96 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 23 Jul 2019 10:42:47 +0800 Subject: [PATCH 12/24] change BUILD_FAISS_WITH_MKL to default off Former-commit-id: 8cf737578611bf2f05cc4da3a775328e6463d341 --- cpp/build.sh | 2 +- cpp/cmake/ThirdPartyPackages.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/build.sh b/cpp/build.sh index ef66a0c375..edfe9305be 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -8,7 +8,7 @@ MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/opt/milvus" PROFILING="OFF" -BUILD_FAISS_WITH_MKL="ON" +BUILD_FAISS_WITH_MKL="OFF" while getopts "p:d:t:uhlrcgm" arg do diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 837d6bd470..fed5dece88 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -700,7 +700,7 @@ endmacro() # FAISS if(NOT DEFINED BUILD_FAISS_WITH_MKL) - set(BUILD_FAISS_WITH_MKL ON) + set(BUILD_FAISS_WITH_MKL OFF) endif() if(EXISTS "/proc/cpuinfo") From 07a9fdb8cadc66ba46057eeb7599e6d59b89abdf Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 23 Jul 2019 15:42:17 +0800 Subject: [PATCH 13/24] MS-246 refine log Former-commit-id: 87ef7f959b43e92c3e7687f9f8e1d059cc353f79 --- cpp/src/db/DBImpl.cpp | 40 +++++++++++++++---- .../sdk/examples/simple/src/ClientTest.cpp | 25 ++++++------ cpp/src/server/RequestTask.cpp | 8 ++++ 3 files changed, 54 insertions(+), 19 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index ccafcbeed9..e99ae99728 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -102,6 +102,7 @@ Status DBImpl::CreateTable(meta::TableSchema& table_schema) { Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { //dates partly delete files of the table but currently we don't support + ENGINE_LOG_DEBUG << "Prepare to delete table " << table_id; mem_mgr_->EraseMemVector(table_id); //not allow insert meta_ptr_->DeleteTable(table_id); //soft delete table @@ -132,6 +133,7 @@ Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count Status DBImpl::InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) { + ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; auto start_time = METRICS_NOW_TIME; Status status = mem_mgr_->InsertVectors(table_id_, n, vectors, vector_ids_); @@ -140,6 +142,8 @@ Status DBImpl::InsertVectors(const std::string& table_id_, // std::chrono::microseconds time_span = std::chrono::duration_cast(end_time - start_time); // double average_time = double(time_span.count()) / n; + ENGINE_LOG_DEBUG << "Insert vectors to cache finished"; + CollectInsertMetrics(total_time, n, status.ok()); return status; @@ -160,6 +164,8 @@ Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { + ENGINE_LOG_DEBUG << "Query by vectors"; + //get all table files from table meta::DatePartionedTableFilesSchema files; auto status = meta_ptr_->FilesToSearch(table_id, dates, files); @@ -181,6 +187,8 @@ Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, Status DBImpl::Query(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { + ENGINE_LOG_DEBUG << "Query by file ids"; + //get specified files std::vector ids; for (auto &id : file_ids) { @@ -269,6 +277,8 @@ void DBImpl::BackgroundTimerTask() { for(auto& iter : index_thread_results_) { iter.wait(); } + + ENGINE_LOG_DEBUG << "DB background thread exit"; break; } @@ -287,6 +297,8 @@ void DBImpl::StartMetricTask() { return; } + ENGINE_LOG_DEBUG << "Start metric task"; + server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity(); @@ -299,20 +311,19 @@ void DBImpl::StartMetricTask() { server::Metrics::GetInstance().GPUPercentGaugeSet(); server::Metrics::GetInstance().GPUMemoryUsageGaugeSet(); server::Metrics::GetInstance().OctetsSet(); + + ENGINE_LOG_DEBUG << "Metric task finished"; } void DBImpl::StartCompactionTask() { -// static int count = 0; -// count++; -// std::cout << "StartCompactionTask: " << count << std::endl; -// std::cout << "c: " << count++ << std::endl; static uint64_t compact_clock_tick = 0; compact_clock_tick++; if(compact_clock_tick%COMPACT_ACTION_INTERVAL != 0) { -// std::cout << "c r: " << count++ << std::endl; return; } + ENGINE_LOG_DEBUG << "Serialize insert cache"; + //serialize memory data std::set temp_table_ids; mem_mgr_->Serialize(temp_table_ids); @@ -320,6 +331,8 @@ void DBImpl::StartCompactionTask() { compact_table_ids_.insert(id); } + ENGINE_LOG_DEBUG << "Insert cache serialized"; + //compactiong has been finished? if(!compact_thread_results_.empty()) { std::chrono::milliseconds span(10); @@ -338,13 +351,15 @@ void DBImpl::StartCompactionTask() { Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files) { + ENGINE_LOG_DEBUG << "Merge files for table" << table_id; + meta::TableFileSchema table_file; table_file.table_id_ = table_id; table_file.date_ = date; Status status = meta_ptr_->CreateTableFile(table_file); if (!status.ok()) { - ENGINE_LOG_INFO << status.ToString() << std::endl; + ENGINE_LOG_ERROR << "Failed to create table: " << status.ToString(); return status; } @@ -396,6 +411,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { meta::DatePartionedTableFilesSchema raw_files; auto status = meta_ptr_->FilesToMerge(table_id, raw_files); if (!status.ok()) { + ENGINE_LOG_ERROR << "Failed to get merge files for table: " << table_id; return status; } @@ -417,12 +433,14 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { } void DBImpl::BackgroundCompaction(std::set table_ids) { + ENGINE_LOG_DEBUG << " Background compaction thread start"; + Status status; for (auto& table_id : table_ids) { status = BackgroundMergeFiles(table_id); if (!status.ok()) { ENGINE_LOG_ERROR << "Merge files for table " << table_id << " failed: " << status.ToString(); - return; + continue;//let other table get chance to merge } } @@ -433,6 +451,8 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { ttl = meta::D_SEC; } meta_ptr_->CleanUpFilesWithTTL(ttl); + + ENGINE_LOG_DEBUG << " Background compaction thread exit"; } void DBImpl::StartBuildIndexTask(bool force) { @@ -477,6 +497,7 @@ Status DBImpl::BuildIndex(const std::string& table_id) { Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { ExecutionEnginePtr to_index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); if(to_index == nullptr) { + ENGINE_LOG_ERROR << "Invalid engine type"; return Status::Error("Invalid engine type"); } @@ -491,6 +512,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { table_file.file_type_ = meta::TableFileSchema::INDEX; //for multi-db-path, distribute index file averagely to each path Status status = meta_ptr_->CreateTableFile(table_file); if (!status.ok()) { + ENGINE_LOG_ERROR << "Failed to create table: " << status.ToString(); return status; } @@ -559,6 +581,8 @@ Status DBImpl::BuildIndexByTable(const std::string& table_id) { } void DBImpl::BackgroundBuildIndex() { + ENGINE_LOG_DEBUG << " Background build index thread start"; + std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; meta_ptr_->FilesToIndex(to_index_files); @@ -574,6 +598,8 @@ void DBImpl::BackgroundBuildIndex() { break; } } + + ENGINE_LOG_DEBUG << " Background build index thread exit"; } Status DBImpl::DropAll() { diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index a17e4e5703..2fb36ea181 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -233,21 +233,22 @@ ClientTest::Test(const std::string& address, const std::string& port) { PrintTableSchema(tb_schema); } + //add vectors std::vector> search_record_array; - {//add vectors - for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors - TimeRecorder recorder("Add vector No." + std::to_string(i)); - std::vector record_array; - int64_t begin_index = i * BATCH_ROW_COUNT; - BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array); - std::vector record_ids; - Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids); - std::cout << "AddVector function call status: " << stat.ToString() << std::endl; - std::cout << "Returned id array count: " << record_ids.size() << std::endl; + for (int i = 0; i < ADD_VECTOR_LOOP; i++) { + TimeRecorder recorder("Add vector No." + std::to_string(i)); + std::vector record_array; + int64_t begin_index = i * BATCH_ROW_COUNT; + BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array); + std::vector record_ids; + Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids); + std::cout << "AddVector function call status: " << stat.ToString() << std::endl; + std::cout << "Returned id array count: " << record_ids.size() << std::endl; - if(search_record_array.size() < NQ) { + if(i == 0) { + for(int64_t k = SEARCH_TARGET; k < SEARCH_TARGET + NQ; k++) { search_record_array.push_back( - std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); + std::make_pair(record_ids[k], record_array[k])); } } } diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index d9e50a82bd..5abd17a13c 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -191,6 +191,7 @@ ServerError CreateTableTask::OnExecute() { } } catch (std::exception& ex) { + SERVER_LOG_ERROR << "CreateTableTask encounter exception: " << ex.what(); return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -236,6 +237,7 @@ ServerError DescribeTableTask::OnExecute() { schema_.store_raw_vector = table_info.store_raw_data_; } catch (std::exception& ex) { + SERVER_LOG_ERROR << "DescribeTableTask encounter exception: " << ex.what(); return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -279,6 +281,7 @@ ServerError BuildIndexTask::OnExecute() { rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { + SERVER_LOG_ERROR << "BuildIndexTask encounter exception: " << ex.what(); return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -316,6 +319,7 @@ ServerError HasTableTask::OnExecute() { rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { + SERVER_LOG_ERROR << "HasTableTask encounter exception: " << ex.what(); return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -365,6 +369,7 @@ ServerError DeleteTableTask::OnExecute() { rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { + SERVER_LOG_ERROR << "DeleteTableTask encounter exception: " << ex.what(); return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -481,6 +486,7 @@ ServerError AddVectorTask::OnExecute() { rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { + SERVER_LOG_ERROR << "AddVectorTask encounter exception: " << ex.what(); return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -604,6 +610,7 @@ ServerError SearchVectorTaskBase::OnExecute() { << " construct result(" << (span_result/total_cost)*100.0 << "%)"; } catch (std::exception& ex) { + SERVER_LOG_ERROR << "SearchVectorTask encounter exception: " << ex.what(); return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -739,6 +746,7 @@ ServerError GetTableRowCountTask::OnExecute() { rc.ElapseFromBegin("totally cost"); } catch (std::exception& ex) { + SERVER_LOG_ERROR << "GetTableRowCountTask encounter exception: " << ex.what(); return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } From 29a177d1299f3f415648878d2871ab2f9cf6f9ee Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Tue, 23 Jul 2019 17:58:11 +0800 Subject: [PATCH 14/24] MS-248 fix profiler build error Former-commit-id: 817200ab3a01e34553090b8074726ea47f8fd7d8 --- cpp/cmake/ThirdPartyPackages.cmake | 9 +++++++-- cpp/src/CMakeLists.txt | 11 ++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index fed5dece88..28180b2c36 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -1676,14 +1676,18 @@ macro(build_gperftools) BUILD_BYPRODUCTS ${GPERFTOOLS_STATIC_LIB}) + ExternalProject_Add_StepDependencies(gperftools_ep build libunwind_ep) + file(MAKE_DIRECTORY "${GPERFTOOLS_INCLUDE_DIR}") - add_library(gperftools SHARED IMPORTED) + add_library(gperftools STATIC IMPORTED) set_target_properties(gperftools PROPERTIES IMPORTED_LOCATION "${GPERFTOOLS_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${GPERFTOOLS_INCLUDE_DIR}") + INTERFACE_INCLUDE_DIRECTORIES "${GPERFTOOLS_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES libunwind) add_dependencies(gperftools gperftools_ep) + add_dependencies(gperftools libunwind_ep) endmacro() if(MILVUS_WITH_GPERFTOOLS) @@ -1692,4 +1696,5 @@ if(MILVUS_WITH_GPERFTOOLS) # TODO: Don't use global includes but rather target_include_directories get_target_property(GPERFTOOLS_INCLUDE_DIR gperftools INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${GPERFTOOLS_INCLUDE_DIR}) + link_directories(SYSTEM ${GPERFTOOLS_PREFIX}/lib) endif() diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 31d1e66c4d..459297a855 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -63,10 +63,6 @@ include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") include_directories(thrift/gen-cpp) include_directories(/usr/include/mysql) -if (MILVUS_ENABLE_PROFILING STREQUAL "ON") - SET(PROFILER_LIB profiler) -endif() - set(third_party_libs easyloggingpp sqlite @@ -85,7 +81,6 @@ set(third_party_libs zlib zstd mysqlpp - ${PROFILER_LIB} ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so cudart ) @@ -103,6 +98,12 @@ else() openblas) endif() +if (MILVUS_ENABLE_PROFILING STREQUAL "ON") + set(third_party_libs ${third_party_libs} + gperftools + libunwind) +endif() + if (GPU_VERSION STREQUAL "ON") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") set(engine_libs From 34ca37d9a0c2e943fd076c6e3a3c7ba740e8e4b9 Mon Sep 17 00:00:00 2001 From: jinhai Date: Wed, 24 Jul 2019 11:02:22 +0800 Subject: [PATCH 15/24] MS-273 Add some logs and reformat code Former-commit-id: 5dfd98a0b954d5cc44ae40f6b14ce65a15d76f80 --- cpp/src/cache/Cache.cpp | 2 +- cpp/src/cache/CacheMgr.cpp | 13 +++ cpp/src/cache/CpuCacheMgr.cpp | 6 +- cpp/src/cache/GpuCacheMgr.cpp | 6 +- cpp/src/config/ConfigNode.cpp | 2 +- cpp/src/db/Constants.h | 12 +-- cpp/src/db/DB.cpp | 3 - cpp/src/db/DB.h | 2 +- cpp/src/db/DBMetaImpl.h | 91 ++++++++++++------- cpp/src/db/IDGenerator.cpp | 2 +- cpp/src/db/IDGenerator.h | 28 +++--- cpp/src/db/Meta.h | 1 + cpp/src/db/MySQLMetaImpl.h | 87 +++++++++--------- cpp/src/db/scheduler/TaskDispatchStrategy.cpp | 3 + cpp/src/db/scheduler/TaskScheduler.cpp | 8 +- 15 files changed, 159 insertions(+), 107 deletions(-) diff --git a/cpp/src/cache/Cache.cpp b/cpp/src/cache/Cache.cpp index a1f2520302..336bf4e82c 100644 --- a/cpp/src/cache/Cache.cpp +++ b/cpp/src/cache/Cache.cpp @@ -89,7 +89,7 @@ void Cache::erase(const std::string& key) { const DataObjPtr& data_ptr = obj_ptr->data_; usage_ -= data_ptr->size(); - SERVER_LOG_DEBUG << "Erase " << key << " from cache"; + SERVER_LOG_DEBUG << "Erase " << key << " size: " << data_ptr->size(); lru_.erase(key); } diff --git a/cpp/src/cache/CacheMgr.cpp b/cpp/src/cache/CacheMgr.cpp index 14efb3a0bb..95deb4c80d 100644 --- a/cpp/src/cache/CacheMgr.cpp +++ b/cpp/src/cache/CacheMgr.cpp @@ -4,6 +4,7 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// +#include "utils/Log.h" #include "CacheMgr.h" #include "metrics/Metrics.h" @@ -20,6 +21,7 @@ CacheMgr::~CacheMgr() { uint64_t CacheMgr::ItemCount() const { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -28,6 +30,7 @@ uint64_t CacheMgr::ItemCount() const { bool CacheMgr::ItemExists(const std::string& key) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return false; } @@ -36,6 +39,7 @@ bool CacheMgr::ItemExists(const std::string& key) { DataObjPtr CacheMgr::GetItem(const std::string& key) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return nullptr; } server::Metrics::GetInstance().CacheAccessTotalIncrement(); @@ -45,6 +49,7 @@ DataObjPtr CacheMgr::GetItem(const std::string& key) { engine::Index_ptr CacheMgr::GetIndex(const std::string& key) { DataObjPtr obj = GetItem(key); if(obj != nullptr) { + SERVER_LOG_ERROR << "Can't get object from key: " << key; return obj->data(); } @@ -53,6 +58,7 @@ engine::Index_ptr CacheMgr::GetIndex(const std::string& key) { void CacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -62,6 +68,7 @@ void CacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) { void CacheMgr::InsertItem(const std::string& key, const engine::Index_ptr& index) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -72,6 +79,7 @@ void CacheMgr::InsertItem(const std::string& key, const engine::Index_ptr& index void CacheMgr::EraseItem(const std::string& key) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -81,6 +89,7 @@ void CacheMgr::EraseItem(const std::string& key) { void CacheMgr::PrintInfo() { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -89,6 +98,7 @@ void CacheMgr::PrintInfo() { void CacheMgr::ClearCache() { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -97,6 +107,7 @@ void CacheMgr::ClearCache() { int64_t CacheMgr::CacheUsage() const { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -105,6 +116,7 @@ int64_t CacheMgr::CacheUsage() const { int64_t CacheMgr::CacheCapacity() const { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -113,6 +125,7 @@ int64_t CacheMgr::CacheCapacity() const { void CacheMgr::SetCapacity(int64_t capacity) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } cache_->set_capacity(capacity); diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index a90f8537b4..167f91f5e5 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -12,10 +12,14 @@ namespace zilliz { namespace milvus { namespace cache { +namespace { + constexpr int64_t unit = 1024 * 1024 * 1024; +} + CpuCacheMgr::CpuCacheMgr() { server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16); - cap *= 1024*1024*1024; + cap *= unit; cache_ = std::make_shared(cap, 1UL<<32); double free_percent = config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85); diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index b3e73fdcdc..13eec4f2b6 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -11,10 +11,14 @@ namespace zilliz { namespace milvus { namespace cache { +namespace { + constexpr int64_t unit = 1024 * 1024 * 1024; +} + GpuCacheMgr::GpuCacheMgr() { server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 1); - cap *= 1024*1024*1024; + cap *= unit; cache_ = std::make_shared(cap, 1UL<<32); } diff --git a/cpp/src/config/ConfigNode.cpp b/cpp/src/config/ConfigNode.cpp index e17eea1daf..a6b91bf67d 100644 --- a/cpp/src/config/ConfigNode.cpp +++ b/cpp/src/config/ConfigNode.cpp @@ -94,7 +94,7 @@ double ConfigNode::GetDoubleValue(const std::string ¶m_key, double default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { - return std::strtold(val.c_str(), nullptr); + return std::strtod(val.c_str(), nullptr); } else { return default_val; } diff --git a/cpp/src/db/Constants.h b/cpp/src/db/Constants.h index 055b10ca9a..e94dfa6aea 100644 --- a/cpp/src/db/Constants.h +++ b/cpp/src/db/Constants.h @@ -9,14 +9,14 @@ namespace zilliz { namespace milvus { namespace engine { -const size_t K = 1024UL; -const size_t M = K * K; -const size_t G = K * M; -const size_t T = K * G; +constexpr size_t K = 1024UL; +constexpr size_t M = K * K; +constexpr size_t G = K * M; +constexpr size_t T = K * G; -const size_t MAX_TABLE_FILE_MEM = 128 * M; +constexpr size_t MAX_TABLE_FILE_MEM = 128 * M; -const int VECTOR_TYPE_SIZE = sizeof(float); +constexpr int VECTOR_TYPE_SIZE = sizeof(float); } // namespace engine } // namespace milvus diff --git a/cpp/src/db/DB.cpp b/cpp/src/db/DB.cpp index ca08bbd9b4..1365d4e116 100644 --- a/cpp/src/db/DB.cpp +++ b/cpp/src/db/DB.cpp @@ -12,11 +12,8 @@ namespace zilliz { namespace milvus { namespace engine { -DB::~DB() {} - void DB::Open(const Options& options, DB** dbptr) { *dbptr = DBFactory::Build(options); - return; } } // namespace engine diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index a416312485..3bfd2281e2 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -52,7 +52,7 @@ public: DB(const DB&) = delete; DB& operator=(const DB&) = delete; - virtual ~DB(); + virtual ~DB() = 0; }; // DB } // namespace engine diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 98a42c000d..6187ad7eae 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -8,67 +8,88 @@ #include "Meta.h" #include "Options.h" + namespace zilliz { namespace milvus { namespace engine { namespace meta { -auto StoragePrototype(const std::string& path); +auto StoragePrototype(const std::string &path); class DBMetaImpl : public Meta { -public: - DBMetaImpl(const DBMetaOptions& options_); + public: + explicit DBMetaImpl(const DBMetaOptions &options_); - virtual Status CreateTable(TableSchema& table_schema) override; - virtual Status DescribeTable(TableSchema& group_info_) override; - virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; - virtual Status AllTables(std::vector& table_schema_array) override; + Status + CreateTable(TableSchema &table_schema) override; - virtual Status DeleteTable(const std::string& table_id) override; - virtual Status DeleteTableFiles(const std::string& table_id) override; + Status + DescribeTable(TableSchema &group_info_) override; - virtual Status CreateTableFile(TableFileSchema& file_schema) override; - virtual Status DropPartitionsByDates(const std::string& table_id, - const DatesT& dates) override; + Status + HasTable(const std::string &table_id, bool &has_or_not) override; - virtual Status GetTableFiles(const std::string& table_id, - const std::vector& ids, - TableFilesSchema& table_files) override; + Status + AllTables(std::vector &table_schema_array) override; - virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override; + Status + DeleteTable(const std::string &table_id) override; - virtual Status UpdateTableFilesToIndex(const std::string& table_id) override; + Status + DeleteTableFiles(const std::string &table_id) override; - virtual Status UpdateTableFile(TableFileSchema& file_schema) override; + Status + CreateTableFile(TableFileSchema &file_schema) override; - virtual Status UpdateTableFiles(TableFilesSchema& files) override; + Status + DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override; - virtual Status FilesToSearch(const std::string& table_id, - const DatesT& partition, - DatePartionedTableFilesSchema& files) override; + Status + GetTableFiles(const std::string &table_id, const std::vector &ids, TableFilesSchema &table_files) override; - virtual Status FilesToMerge(const std::string& table_id, - DatePartionedTableFilesSchema& files) override; + Status + HasNonIndexFiles(const std::string &table_id, bool &has) override; - virtual Status FilesToIndex(TableFilesSchema&) override; + Status + UpdateTableFilesToIndex(const std::string &table_id) override; - virtual Status Archive() override; + Status + UpdateTableFile(TableFileSchema &file_schema) override; - virtual Status Size(uint64_t& result) override; + Status + UpdateTableFiles(TableFilesSchema &files) override; - virtual Status CleanUp() override; + Status + FilesToSearch(const std::string &table_id, const DatesT &partition, DatePartionedTableFilesSchema &files) override; - virtual Status CleanUpFilesWithTTL(uint16_t seconds) override; + Status + FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) override; - virtual Status DropAll() override; + Status + FilesToIndex(TableFilesSchema &) override; - virtual Status Count(const std::string& table_id, uint64_t& result) override; + Status + Archive() override; - virtual ~DBMetaImpl(); + Status + Size(uint64_t &result) override; -private: - Status NextFileId(std::string& file_id); - Status NextTableId(std::string& table_id); + Status + CleanUp() override; + + Status + CleanUpFilesWithTTL(uint16_t seconds) override; + + Status + DropAll() override; + + Status Count(const std::string &table_id, uint64_t &result) override; + + ~DBMetaImpl() override; + + private: + Status NextFileId(std::string &file_id); + Status NextTableId(std::string &table_id); Status DiscardFiles(long to_discard_size); Status Initialize(); diff --git a/cpp/src/db/IDGenerator.cpp b/cpp/src/db/IDGenerator.cpp index c4f9787ba7..70d71817fc 100644 --- a/cpp/src/db/IDGenerator.cpp +++ b/cpp/src/db/IDGenerator.cpp @@ -13,7 +13,7 @@ namespace zilliz { namespace milvus { namespace engine { -IDGenerator::~IDGenerator() {} +constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO; IDNumber SimpleIDGenerator::GetNextIDNumber() { auto now = std::chrono::system_clock::now(); diff --git a/cpp/src/db/IDGenerator.h b/cpp/src/db/IDGenerator.h index 079485060c..502b2865f1 100644 --- a/cpp/src/db/IDGenerator.h +++ b/cpp/src/db/IDGenerator.h @@ -10,28 +10,34 @@ #include #include + namespace zilliz { namespace milvus { namespace engine { class IDGenerator { -public: + public: virtual IDNumber GetNextIDNumber() = 0; - virtual void GetNextIDNumbers(size_t n, IDNumbers& ids) = 0; - - virtual ~IDGenerator(); - + virtual void GetNextIDNumbers(size_t n, IDNumbers &ids) = 0; + virtual ~IDGenerator() = 0; }; // IDGenerator class SimpleIDGenerator : public IDGenerator { -public: - virtual IDNumber GetNextIDNumber() override; - virtual void GetNextIDNumbers(size_t n, IDNumbers& ids) override; + public: + ~SimpleIDGenerator() override = default; -private: - void NextIDNumbers(size_t n, IDNumbers& ids); - const size_t MAX_IDS_PER_MICRO = 1000; + IDNumber + GetNextIDNumber() override; + + void + GetNextIDNumbers(size_t n, IDNumbers &ids) override; + + private: + void + NextIDNumbers(size_t n, IDNumbers &ids); + + static constexpr size_t MAX_IDS_PER_MICRO = 1000; }; // SimpleIDGenerator diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 1d83817f5d..b18cea35e1 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -23,6 +23,7 @@ class Meta { public: using Ptr = std::shared_ptr; + virtual ~Meta() = 0; virtual Status CreateTable(TableSchema& table_schema) = 0; virtual Status DescribeTable(TableSchema& table_schema) = 0; virtual Status HasTable(const std::string& table_id, bool& has_or_not) = 0; diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index 403d5b0b59..87bc1783c7 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -12,79 +12,80 @@ #include "mysql++/mysql++.h" #include + namespace zilliz { namespace milvus { namespace engine { namespace meta { // auto StoragePrototype(const std::string& path); - using namespace mysqlpp; +using namespace mysqlpp; - class MySQLMetaImpl : public Meta { - public: - MySQLMetaImpl(const DBMetaOptions& options_, const int& mode); +class MySQLMetaImpl : public Meta { + public: + MySQLMetaImpl(const DBMetaOptions &options_, const int &mode); - virtual Status CreateTable(TableSchema& table_schema) override; - virtual Status DescribeTable(TableSchema& group_info_) override; - virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; - virtual Status AllTables(std::vector& table_schema_array) override; + Status CreateTable(TableSchema &table_schema) override; + Status DescribeTable(TableSchema &group_info_) override; + Status HasTable(const std::string &table_id, bool &has_or_not) override; + Status AllTables(std::vector &table_schema_array) override; - virtual Status DeleteTable(const std::string& table_id) override; - virtual Status DeleteTableFiles(const std::string& table_id) override; + Status DeleteTable(const std::string &table_id) override; + Status DeleteTableFiles(const std::string &table_id) override; - virtual Status CreateTableFile(TableFileSchema& file_schema) override; - virtual Status DropPartitionsByDates(const std::string& table_id, - const DatesT& dates) override; + Status CreateTableFile(TableFileSchema &file_schema) override; + Status DropPartitionsByDates(const std::string &table_id, + const DatesT &dates) override; - virtual Status GetTableFiles(const std::string& table_id, - const std::vector& ids, - TableFilesSchema& table_files) override; + Status GetTableFiles(const std::string &table_id, + const std::vector &ids, + TableFilesSchema &table_files) override; - virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override; + Status HasNonIndexFiles(const std::string &table_id, bool &has) override; - virtual Status UpdateTableFile(TableFileSchema& file_schema) override; + Status UpdateTableFile(TableFileSchema &file_schema) override; - virtual Status UpdateTableFilesToIndex(const std::string& table_id) override; + Status UpdateTableFilesToIndex(const std::string &table_id) override; - virtual Status UpdateTableFiles(TableFilesSchema& files) override; + Status UpdateTableFiles(TableFilesSchema &files) override; - virtual Status FilesToSearch(const std::string& table_id, - const DatesT& partition, - DatePartionedTableFilesSchema& files) override; + Status FilesToSearch(const std::string &table_id, + const DatesT &partition, + DatePartionedTableFilesSchema &files) override; - virtual Status FilesToMerge(const std::string& table_id, - DatePartionedTableFilesSchema& files) override; + Status FilesToMerge(const std::string &table_id, + DatePartionedTableFilesSchema &files) override; - virtual Status FilesToIndex(TableFilesSchema&) override; + Status FilesToIndex(TableFilesSchema &) override; - virtual Status Archive() override; + Status Archive() override; - virtual Status Size(uint64_t& result) override; + Status Size(uint64_t &result) override; - virtual Status CleanUp() override; + Status CleanUp() override; - virtual Status CleanUpFilesWithTTL(uint16_t seconds) override; + Status CleanUpFilesWithTTL(uint16_t seconds) override; - virtual Status DropAll() override; + Status DropAll() override; - virtual Status Count(const std::string& table_id, uint64_t& result) override; + Status Count(const std::string &table_id, uint64_t &result) override; - virtual ~MySQLMetaImpl(); + virtual ~MySQLMetaImpl(); - private: - Status NextFileId(std::string& file_id); - Status NextTableId(std::string& table_id); - Status DiscardFiles(long long to_discard_size); - Status Initialize(); + private: + Status NextFileId(std::string &file_id); + Status NextTableId(std::string &table_id); + Status DiscardFiles(long long to_discard_size); + Status Initialize(); - const DBMetaOptions options_; - const int mode_; + const DBMetaOptions options_; + const int mode_; - std::shared_ptr mysql_connection_pool_; - bool safe_grab = false; + std::shared_ptr mysql_connection_pool_; + bool safe_grab = false; // std::mutex connectionMutex_; - }; // DBMetaImpl +}; // DBMetaImpl } // namespace meta } // namespace engine diff --git a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp index 73c46942c8..38c9463117 100644 --- a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp +++ b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp @@ -20,6 +20,7 @@ class ReuseCacheIndexStrategy { public: bool Schedule(const SearchContextPtr &context, std::list& task_list) { if(context == nullptr) { + ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist"; return false; } @@ -64,6 +65,7 @@ class DeleteTableStrategy { public: bool Schedule(const DeleteContextPtr &context, std::list &task_list) { if (context == nullptr) { + ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist"; return false; } @@ -103,6 +105,7 @@ public: bool TaskDispatchStrategy::Schedule(const ScheduleContextPtr &context_ptr, std::list &task_list) { if(context_ptr == nullptr) { + ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist"; return false; } diff --git a/cpp/src/db/scheduler/TaskScheduler.cpp b/cpp/src/db/scheduler/TaskScheduler.cpp index 04d6762aae..f29c3d4caa 100644 --- a/cpp/src/db/scheduler/TaskScheduler.cpp +++ b/cpp/src/db/scheduler/TaskScheduler.cpp @@ -31,6 +31,7 @@ TaskScheduler& TaskScheduler::GetInstance() { bool TaskScheduler::Start() { if(!stopped_) { + SERVER_LOG_INFO << "Task Scheduler isn't started"; return true; } @@ -47,6 +48,7 @@ TaskScheduler::Start() { bool TaskScheduler::Stop() { if(stopped_) { + SERVER_LOG_INFO << "Task Scheduler already stopped"; return true; } @@ -80,7 +82,7 @@ TaskScheduler::TaskDispatchWorker() { ScheduleTaskPtr task_ptr = task_dispatch_queue_.Take(); if(task_ptr == nullptr) { SERVER_LOG_INFO << "Stop db task dispatch thread"; - break;//exit + return true; } //execute task @@ -98,8 +100,8 @@ TaskScheduler::TaskWorker() { while(true) { ScheduleTaskPtr task_ptr = task_queue_.Take(); if(task_ptr == nullptr) { - SERVER_LOG_INFO << "Stop db task thread"; - break;//exit + SERVER_LOG_INFO << "Stop db task worker thread"; + return true; } //execute task From aee7809c6171290fe9f6a9c2597849a799568962 Mon Sep 17 00:00:00 2001 From: jinhai Date: Wed, 24 Jul 2019 11:17:53 +0800 Subject: [PATCH 16/24] MS-273 Add some logs and reformat code, fix compile error Former-commit-id: 5bf635ab31d5dfd0cdb558b7f038670098d75ed0 --- cpp/src/db/DB.cpp | 2 + cpp/src/db/IDGenerator.cpp | 2 + cpp/src/db/IDGenerator.h | 11 +++-- cpp/src/db/Meta.cpp | 2 + cpp/src/db/Meta.h | 95 +++++++++++++++++++++++++------------- 5 files changed, 76 insertions(+), 36 deletions(-) diff --git a/cpp/src/db/DB.cpp b/cpp/src/db/DB.cpp index 1365d4e116..3b4f03d247 100644 --- a/cpp/src/db/DB.cpp +++ b/cpp/src/db/DB.cpp @@ -12,6 +12,8 @@ namespace zilliz { namespace milvus { namespace engine { +DB::~DB() = default; + void DB::Open(const Options& options, DB** dbptr) { *dbptr = DBFactory::Build(options); } diff --git a/cpp/src/db/IDGenerator.cpp b/cpp/src/db/IDGenerator.cpp index 70d71817fc..74d3a1433a 100644 --- a/cpp/src/db/IDGenerator.cpp +++ b/cpp/src/db/IDGenerator.cpp @@ -13,6 +13,8 @@ namespace zilliz { namespace milvus { namespace engine { +IDGenerator::~IDGenerator() = default; + constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO; IDNumber SimpleIDGenerator::GetNextIDNumber() { diff --git a/cpp/src/db/IDGenerator.h b/cpp/src/db/IDGenerator.h index 502b2865f1..eb39221734 100644 --- a/cpp/src/db/IDGenerator.h +++ b/cpp/src/db/IDGenerator.h @@ -17,9 +17,14 @@ namespace engine { class IDGenerator { public: - virtual IDNumber GetNextIDNumber() = 0; - virtual void GetNextIDNumbers(size_t n, IDNumbers &ids) = 0; - virtual ~IDGenerator() = 0; + virtual + IDNumber GetNextIDNumber() = 0; + + virtual void + GetNextIDNumbers(size_t n, IDNumbers &ids) = 0; + + virtual + ~IDGenerator() = 0; }; // IDGenerator diff --git a/cpp/src/db/Meta.cpp b/cpp/src/db/Meta.cpp index cc7525528f..a86051a1c4 100644 --- a/cpp/src/db/Meta.cpp +++ b/cpp/src/db/Meta.cpp @@ -13,6 +13,8 @@ namespace milvus { namespace engine { namespace meta { +Meta::~Meta() = default; + DateT Meta::GetDate(const std::time_t& t, int day_delta) { struct tm ltm; localtime_r(&t, <m); diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index b18cea35e1..5275605611 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -20,57 +20,86 @@ namespace meta { class Meta { -public: + public: using Ptr = std::shared_ptr; - virtual ~Meta() = 0; - virtual Status CreateTable(TableSchema& table_schema) = 0; - virtual Status DescribeTable(TableSchema& table_schema) = 0; - virtual Status HasTable(const std::string& table_id, bool& has_or_not) = 0; - virtual Status AllTables(std::vector& table_schema_array) = 0; + virtual + ~Meta() = 0; - virtual Status DeleteTable(const std::string& table_id) = 0; - virtual Status DeleteTableFiles(const std::string& table_id) = 0; + virtual Status + CreateTable(TableSchema &table_schema) = 0; - virtual Status CreateTableFile(TableFileSchema& file_schema) = 0; - virtual Status DropPartitionsByDates(const std::string& table_id, - const DatesT& dates) = 0; + virtual Status + DescribeTable(TableSchema &table_schema) = 0; - virtual Status GetTableFiles(const std::string& table_id, - const std::vector& ids, - TableFilesSchema& table_files) = 0; + virtual Status + HasTable(const std::string &table_id, bool &has_or_not) = 0; - virtual Status UpdateTableFilesToIndex(const std::string& table_id) = 0; + virtual Status + AllTables(std::vector &table_schema_array) = 0; - virtual Status UpdateTableFile(TableFileSchema& file_schema) = 0; + virtual Status + DeleteTable(const std::string &table_id) = 0; - virtual Status UpdateTableFiles(TableFilesSchema& files) = 0; + virtual Status + DeleteTableFiles(const std::string &table_id) = 0; - virtual Status FilesToSearch(const std::string &table_id, - const DatesT &partition, - DatePartionedTableFilesSchema& files) = 0; + virtual Status + CreateTableFile(TableFileSchema &file_schema) = 0; - virtual Status FilesToMerge(const std::string& table_id, - DatePartionedTableFilesSchema& files) = 0; + virtual Status + DropPartitionsByDates(const std::string &table_id, const DatesT &dates) = 0; - virtual Status Size(uint64_t& result) = 0; + virtual Status + GetTableFiles(const std::string &table_id, const std::vector &ids, TableFilesSchema &table_files) = 0; - virtual Status Archive() = 0; + virtual Status + UpdateTableFilesToIndex(const std::string &table_id) = 0; - virtual Status FilesToIndex(TableFilesSchema&) = 0; + virtual Status + UpdateTableFile(TableFileSchema &file_schema) = 0; - virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) = 0; + virtual Status + UpdateTableFiles(TableFilesSchema &files) = 0; - virtual Status CleanUp() = 0; - virtual Status CleanUpFilesWithTTL(uint16_t) = 0; + virtual Status + FilesToSearch(const std::string &table_id, const DatesT &partition, DatePartionedTableFilesSchema &files) = 0; - virtual Status DropAll() = 0; + virtual Status + FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) = 0; - virtual Status Count(const std::string& table_id, uint64_t& result) = 0; + virtual Status + Size(uint64_t &result) = 0; - static DateT GetDate(const std::time_t& t, int day_delta = 0); - static DateT GetDate(); - static DateT GetDateWithDelta(int day_delta); + virtual Status + Archive() = 0; + + virtual Status + FilesToIndex(TableFilesSchema &) = 0; + + virtual Status + HasNonIndexFiles(const std::string &table_id, bool &has) = 0; + + virtual Status + CleanUp() = 0; + + virtual Status + CleanUpFilesWithTTL(uint16_t) = 0; + + virtual Status + DropAll() = 0; + + virtual Status + Count(const std::string &table_id, uint64_t &result) = 0; + + static DateT + GetDate(const std::time_t &t, int day_delta = 0); + + static DateT + GetDate(); + + static DateT + GetDateWithDelta(int day_delta); }; // MetaData From ed029178c4466d12e626917b382b815d1f988646 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 24 Jul 2019 12:48:36 +0800 Subject: [PATCH 17/24] MS-266 Improve topk reduce time by using multi-threads Former-commit-id: 79e4cfe6ade7b0fc059cd246c4b670a4b5343ca3 --- cpp/conf/server_config.template | 2 + cpp/src/db/DBImpl.cpp | 18 +-- cpp/src/db/scheduler/task/SearchTask.cpp | 108 +++++++++++--- cpp/src/server/ServerConfig.h | 1 + cpp/unittest/db/search_test.cpp | 174 +++++++++++++++++++---- 5 files changed, 247 insertions(+), 56 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 2942ffa179..8c81315de9 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -8,6 +8,8 @@ db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path db_slave_path: # secondry data storage path, split by semicolon + parallel_reduce: true # use multi-threads to reduce topk result + # URI format: dialect://username:password@host:port/database # All parts except dialect are optional, but you MUST include the delimiters # Currently dialect supports mysql or sqlite diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index e99ae99728..ce3af88e2c 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -297,7 +297,7 @@ void DBImpl::StartMetricTask() { return; } - ENGINE_LOG_DEBUG << "Start metric task"; + ENGINE_LOG_INFO << "Start metric task"; server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); @@ -312,7 +312,7 @@ void DBImpl::StartMetricTask() { server::Metrics::GetInstance().GPUMemoryUsageGaugeSet(); server::Metrics::GetInstance().OctetsSet(); - ENGINE_LOG_DEBUG << "Metric task finished"; + ENGINE_LOG_INFO << "Metric task finished"; } void DBImpl::StartCompactionTask() { @@ -322,8 +322,6 @@ void DBImpl::StartCompactionTask() { return; } - ENGINE_LOG_DEBUG << "Serialize insert cache"; - //serialize memory data std::set temp_table_ids; mem_mgr_->Serialize(temp_table_ids); @@ -331,7 +329,9 @@ void DBImpl::StartCompactionTask() { compact_table_ids_.insert(id); } - ENGINE_LOG_DEBUG << "Insert cache serialized"; + if(!temp_table_ids.empty()) { + SERVER_LOG_DEBUG << "Insert cache serialized"; + } //compactiong has been finished? if(!compact_thread_results_.empty()) { @@ -433,7 +433,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { } void DBImpl::BackgroundCompaction(std::set table_ids) { - ENGINE_LOG_DEBUG << " Background compaction thread start"; + ENGINE_LOG_INFO << " Background compaction thread start"; Status status; for (auto& table_id : table_ids) { @@ -452,7 +452,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { } meta_ptr_->CleanUpFilesWithTTL(ttl); - ENGINE_LOG_DEBUG << " Background compaction thread exit"; + ENGINE_LOG_INFO << " Background compaction thread exit"; } void DBImpl::StartBuildIndexTask(bool force) { @@ -581,7 +581,7 @@ Status DBImpl::BuildIndexByTable(const std::string& table_id) { } void DBImpl::BackgroundBuildIndex() { - ENGINE_LOG_DEBUG << " Background build index thread start"; + ENGINE_LOG_INFO << " Background build index thread start"; std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; @@ -599,7 +599,7 @@ void DBImpl::BackgroundBuildIndex() { } } - ENGINE_LOG_DEBUG << " Background build index thread exit"; + ENGINE_LOG_INFO << " Background build index thread exit"; } Status DBImpl::DropAll() { diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 1c0776ee89..e696faaed0 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -5,14 +5,60 @@ ******************************************************************************/ #include "SearchTask.h" #include "metrics/Metrics.h" -#include "utils/Log.h" +#include "db/Log.h" #include "utils/TimeRecorder.h" +#include + namespace zilliz { namespace milvus { namespace engine { namespace { + +static constexpr size_t PARALLEL_REDUCE_THRESHOLD = 10000; +static constexpr size_t PARALLEL_REDUCE_BATCH = 1000; + +bool NeedParallelReduce(uint64_t nq, uint64_t topk) { + server::ServerConfig &config = server::ServerConfig::GetInstance(); + server::ConfigNode& db_config = config.GetConfig(server::CONFIG_DB); + bool need_parallel = db_config.GetBoolValue(server::CONFIG_DB_PARALLEL_REDUCE, true); + if(!need_parallel) { + return false; + } + + return nq*topk >= PARALLEL_REDUCE_THRESHOLD; +} + +void ParallelReduce(std::function& reduce_function, size_t max_index) { + size_t reduce_batch = PARALLEL_REDUCE_BATCH; + + auto thread_count = std::thread::hardware_concurrency() - 1; //not all core do this work + if(thread_count > 0) { + reduce_batch = max_index/thread_count + 1; + } + ENGINE_LOG_DEBUG << "use " << thread_count << + " thread parallelly do reduce, each thread process " << reduce_batch << " vectors"; + + std::vector > thread_array; + size_t from_index = 0; + while(from_index < max_index) { + size_t to_index = from_index + reduce_batch; + if(to_index > max_index) { + to_index = max_index; + } + + auto reduce_thread = std::make_shared(reduce_function, from_index, to_index); + thread_array.push_back(reduce_thread); + + from_index = to_index; + } + + for(auto& thread_ptr : thread_array) { + thread_ptr->join(); + } +} + void CollectDurationMetrics(int index_type, double total_time) { switch(index_type) { case meta::TableFileSchema::RAW: { @@ -32,7 +78,7 @@ void CollectDurationMetrics(int index_type, double total_time) { std::string GetMetricType() { server::ServerConfig &config = server::ServerConfig::GetInstance(); - server::ConfigNode engine_config = config.GetConfig(server::CONFIG_ENGINE); + server::ConfigNode& engine_config = config.GetConfig(server::CONFIG_ENGINE); return engine_config.GetValue(server::CONFIG_METRICTYPE, "L2"); } @@ -51,7 +97,7 @@ std::shared_ptr SearchTask::Execute() { return nullptr; } - SERVER_LOG_DEBUG << "Searching in file id:" << index_id_<< " with " + ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_<< " with " << search_contexts_.size() << " tasks"; server::TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_)); @@ -79,6 +125,9 @@ std::shared_ptr SearchTask::Execute() { auto spec_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk(); SearchTask::ClusterResult(output_ids, output_distence, context->nq(), spec_k, result_set); + span = rc.RecordSection("cluster result for context:" + context->Identity()); + context->AccumReduceCost(span); + //step 4: pick up topk result SearchTask::TopkResult(result_set, inner_k, metric_l2, context->GetResult()); @@ -86,7 +135,7 @@ std::shared_ptr SearchTask::Execute() { context->AccumReduceCost(span); } catch (std::exception& ex) { - SERVER_LOG_ERROR << "SearchTask encounter exception: " << ex.what(); + ENGINE_LOG_ERROR << "SearchTask encounter exception: " << ex.what(); context->IndexSearchDone(index_id_);//mark as done avoid dead lock, even search failed continue; } @@ -112,23 +161,32 @@ Status SearchTask::ClusterResult(const std::vector &output_ids, if(output_ids.size() < nq*topk || output_distence.size() < nq*topk) { std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + " distance array size: " + std::to_string(output_distence.size()); - SERVER_LOG_ERROR << msg; + ENGINE_LOG_ERROR << msg; return Status::Error(msg); } result_set.clear(); - result_set.reserve(nq); - for (auto i = 0; i < nq; i++) { - SearchContext::Id2DistanceMap id_distance; - id_distance.reserve(topk); - for (auto k = 0; k < topk; k++) { - uint64_t index = i * topk + k; - if(output_ids[index] < 0) { - continue; + result_set.resize(nq); + + std::function reduce_worker = [&](size_t from_index, size_t to_index) { + for (auto i = from_index; i < to_index; i++) { + SearchContext::Id2DistanceMap id_distance; + id_distance.reserve(topk); + for (auto k = 0; k < topk; k++) { + uint64_t index = i * topk + k; + if(output_ids[index] < 0) { + continue; + } + id_distance.push_back(std::make_pair(output_ids[index], output_distence[index])); } - id_distance.push_back(std::make_pair(output_ids[index], output_distence[index])); + result_set[i] = id_distance; } - result_set.emplace_back(id_distance); + }; + + if(NeedParallelReduce(nq, topk)) { + ParallelReduce(reduce_worker, nq); + } else { + reduce_worker(0, nq); } return Status::OK(); @@ -140,7 +198,7 @@ Status SearchTask::MergeResult(SearchContext::Id2DistanceMap &distance_src, bool ascending) { //Note: the score_src and score_target are already arranged by score in ascending order if(distance_src.empty()) { - SERVER_LOG_WARNING << "Empty distance source array"; + ENGINE_LOG_WARNING << "Empty distance source array"; return Status::OK(); } @@ -218,14 +276,22 @@ Status SearchTask::TopkResult(SearchContext::ResultSet &result_src, if (result_src.size() != result_target.size()) { std::string msg = "Invalid result set size"; - SERVER_LOG_ERROR << msg; + ENGINE_LOG_ERROR << msg; return Status::Error(msg); } - for (size_t i = 0; i < result_src.size(); i++) { - SearchContext::Id2DistanceMap &score_src = result_src[i]; - SearchContext::Id2DistanceMap &score_target = result_target[i]; - SearchTask::MergeResult(score_src, score_target, topk, ascending); + std::function ReduceWorker = [&](size_t from_index, size_t to_index) { + for (size_t i = from_index; i < to_index; i++) { + SearchContext::Id2DistanceMap &score_src = result_src[i]; + SearchContext::Id2DistanceMap &score_target = result_target[i]; + SearchTask::MergeResult(score_src, score_target, topk, ascending); + } + }; + + if(NeedParallelReduce(result_src.size(), topk)) { + ParallelReduce(ReduceWorker, result_src.size()); + } else { + ReduceWorker(0, result_src.size()); } return Status::OK(); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index 4a1b089810..bb7d5d3669 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -29,6 +29,7 @@ static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshol static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; static const std::string CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; +static const std::string CONFIG_DB_PARALLEL_REDUCE = "parallel_reduce"; static const std::string CONFIG_LOG = "log_config"; diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index c4ee7a48b5..98f2c88ea0 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -6,6 +6,8 @@ #include #include "db/scheduler/task/SearchTask.h" +#include "utils/TimeRecorder.h" + #include #include @@ -17,27 +19,33 @@ static constexpr uint64_t NQ = 15; static constexpr uint64_t TOP_K = 64; void BuildResult(uint64_t nq, - uint64_t top_k, + uint64_t topk, + bool ascending, std::vector &output_ids, std::vector &output_distence) { output_ids.clear(); - output_ids.resize(nq*top_k); + output_ids.resize(nq*topk); output_distence.clear(); - output_distence.resize(nq*top_k); + output_distence.resize(nq*topk); for(uint64_t i = 0; i < nq; i++) { - for(uint64_t j = 0; j < top_k; j++) { - output_ids[i * top_k + j] = (long)(drand48()*100000); - output_distence[i * top_k + j] = j + drand48(); + for(uint64_t j = 0; j < topk; j++) { + output_ids[i * topk + j] = (long)(drand48()*100000); + output_distence[i * topk + j] = ascending ? (j + drand48()) : ((topk - j) + drand48()); } } } void CheckResult(const engine::SearchContext::Id2DistanceMap& src_1, const engine::SearchContext::Id2DistanceMap& src_2, - const engine::SearchContext::Id2DistanceMap& target) { + const engine::SearchContext::Id2DistanceMap& target, + bool ascending) { for(uint64_t i = 0; i < target.size() - 1; i++) { - ASSERT_LE(target[i].second, target[i + 1].second); + if(ascending) { + ASSERT_LE(target[i].second, target[i + 1].second); + } else { + ASSERT_GE(target[i].second, target[i + 1].second); + } } using ID2DistMap = std::map; @@ -57,9 +65,52 @@ void CheckResult(const engine::SearchContext::Id2DistanceMap& src_1, } } +void CheckCluster(const std::vector& target_ids, + const std::vector& target_distence, + const engine::SearchContext::ResultSet& src_result, + int64_t nq, + int64_t topk) { + ASSERT_EQ(src_result.size(), nq); + for(int64_t i = 0; i < nq; i++) { + auto& res = src_result[i]; + ASSERT_EQ(res.size(), topk); + + if(res.empty()) { + continue; + } + + ASSERT_EQ(res[0].first, target_ids[i*topk]); + ASSERT_EQ(res[topk - 1].first, target_ids[i*topk + topk - 1]); + } +} + +void CheckTopkResult(const engine::SearchContext::ResultSet& src_result, + bool ascending, + int64_t nq, + int64_t topk) { + ASSERT_EQ(src_result.size(), nq); + for(int64_t i = 0; i < nq; i++) { + auto& res = src_result[i]; + ASSERT_EQ(res.size(), topk); + + if(res.empty()) { + continue; + } + + for(int64_t k = 0; k < topk - 1; k++) { + if(ascending) { + ASSERT_LE(res[k].second, res[k + 1].second); + } else { + ASSERT_GE(res[k].second, res[k + 1].second); + } + } + } +} + } TEST(DBSearchTest, TOPK_TEST) { + bool ascending = true; std::vector target_ids; std::vector target_distence; engine::SearchContext::ResultSet src_result; @@ -67,19 +118,19 @@ TEST(DBSearchTest, TOPK_TEST) { ASSERT_FALSE(status.ok()); ASSERT_TRUE(src_result.empty()); - BuildResult(NQ, TOP_K, target_ids, target_distence); + BuildResult(NQ, TOP_K, ascending, target_ids, target_distence); status = engine::SearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); ASSERT_TRUE(status.ok()); ASSERT_EQ(src_result.size(), NQ); engine::SearchContext::ResultSet target_result; - status = engine::SearchTask::TopkResult(target_result, TOP_K, true, target_result); + status = engine::SearchTask::TopkResult(target_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); - status = engine::SearchTask::TopkResult(target_result, TOP_K, true, src_result); + status = engine::SearchTask::TopkResult(target_result, TOP_K, ascending, src_result); ASSERT_FALSE(status.ok()); - status = engine::SearchTask::TopkResult(src_result, TOP_K, true, target_result); + status = engine::SearchTask::TopkResult(src_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); ASSERT_TRUE(src_result.empty()); ASSERT_EQ(target_result.size(), NQ); @@ -87,21 +138,21 @@ TEST(DBSearchTest, TOPK_TEST) { std::vector src_ids; std::vector src_distence; uint64_t wrong_topk = TOP_K - 10; - BuildResult(NQ, wrong_topk, src_ids, src_distence); + BuildResult(NQ, wrong_topk, ascending, src_ids, src_distence); status = engine::SearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result); ASSERT_TRUE(status.ok()); - status = engine::SearchTask::TopkResult(src_result, TOP_K, true, target_result); + status = engine::SearchTask::TopkResult(src_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); for(uint64_t i = 0; i < NQ; i++) { ASSERT_EQ(target_result[i].size(), TOP_K); } wrong_topk = TOP_K + 10; - BuildResult(NQ, wrong_topk, src_ids, src_distence); + BuildResult(NQ, wrong_topk, ascending, src_ids, src_distence); - status = engine::SearchTask::TopkResult(src_result, TOP_K, true, target_result); + status = engine::SearchTask::TopkResult(src_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); for(uint64_t i = 0; i < NQ; i++) { ASSERT_EQ(target_result[i].size(), TOP_K); @@ -109,6 +160,7 @@ TEST(DBSearchTest, TOPK_TEST) { } TEST(DBSearchTest, MERGE_TEST) { + bool ascending = true; std::vector target_ids; std::vector target_distence; std::vector src_ids; @@ -116,8 +168,8 @@ TEST(DBSearchTest, MERGE_TEST) { engine::SearchContext::ResultSet src_result, target_result; uint64_t src_count = 5, target_count = 8; - BuildResult(1, src_count, src_ids, src_distence); - BuildResult(1, target_count, target_ids, target_distence); + BuildResult(1, src_count, ascending, src_ids, src_distence); + BuildResult(1, target_count, ascending, target_ids, target_distence); auto status = engine::SearchTask::ClusterResult(src_ids, src_distence, 1, src_count, src_result); ASSERT_TRUE(status.ok()); status = engine::SearchTask::ClusterResult(target_ids, target_distence, 1, target_count, target_result); @@ -126,37 +178,107 @@ TEST(DBSearchTest, MERGE_TEST) { { engine::SearchContext::Id2DistanceMap src = src_result[0]; engine::SearchContext::Id2DistanceMap target = target_result[0]; - status = engine::SearchTask::MergeResult(src, target, 10, true); + status = engine::SearchTask::MergeResult(src, target, 10, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), 10); - CheckResult(src_result[0], target_result[0], target); + CheckResult(src_result[0], target_result[0], target, ascending); } { engine::SearchContext::Id2DistanceMap src = src_result[0]; engine::SearchContext::Id2DistanceMap target; - status = engine::SearchTask::MergeResult(src, target, 10, true); + status = engine::SearchTask::MergeResult(src, target, 10, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count); ASSERT_TRUE(src.empty()); - CheckResult(src_result[0], target_result[0], target); + CheckResult(src_result[0], target_result[0], target, ascending); } { engine::SearchContext::Id2DistanceMap src = src_result[0]; engine::SearchContext::Id2DistanceMap target = target_result[0]; - status = engine::SearchTask::MergeResult(src, target, 30, true); + status = engine::SearchTask::MergeResult(src, target, 30, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count + target_count); - CheckResult(src_result[0], target_result[0], target); + CheckResult(src_result[0], target_result[0], target, ascending); } { engine::SearchContext::Id2DistanceMap target = src_result[0]; engine::SearchContext::Id2DistanceMap src = target_result[0]; - status = engine::SearchTask::MergeResult(src, target, 30, true); + status = engine::SearchTask::MergeResult(src, target, 30, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count + target_count); - CheckResult(src_result[0], target_result[0], target); + CheckResult(src_result[0], target_result[0], target, ascending); } } + +TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) { + bool ascending = true; + std::vector target_ids; + std::vector target_distence; + engine::SearchContext::ResultSet src_result; + + auto DoCluster = [&](int64_t nq, int64_t topk) { + server::TimeRecorder rc("DoCluster"); + src_result.clear(); + BuildResult(nq, topk, ascending, target_ids, target_distence); + rc.RecordSection("build id/dietance map"); + + auto status = engine::SearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(src_result.size(), nq); + + rc.RecordSection("cluster result"); + + CheckCluster(target_ids, target_distence, src_result, nq, topk); + rc.RecordSection("check result"); + }; + + DoCluster(10000, 1000); + DoCluster(333, 999); + DoCluster(1, 1000); + DoCluster(1, 1); + DoCluster(7, 0); + DoCluster(9999, 1); + DoCluster(10001, 1); + DoCluster(58273, 1234); +} + +TEST(DBSearchTest, PARALLEL_TOPK_TEST) { + std::vector target_ids; + std::vector target_distence; + engine::SearchContext::ResultSet src_result; + + std::vector insufficient_ids; + std::vector insufficient_distence; + engine::SearchContext::ResultSet insufficient_result; + + auto DoTopk = [&](int64_t nq, int64_t topk,int64_t insufficient_topk, bool ascending) { + src_result.clear(); + insufficient_result.clear(); + + server::TimeRecorder rc("DoCluster"); + + BuildResult(nq, topk, ascending, target_ids, target_distence); + auto status = engine::SearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); + rc.RecordSection("cluster result"); + + BuildResult(nq, insufficient_topk, ascending, insufficient_ids, insufficient_distence); + status = engine::SearchTask::ClusterResult(target_ids, target_distence, nq, insufficient_topk, insufficient_result); + rc.RecordSection("cluster result"); + + engine::SearchTask::TopkResult(insufficient_result, topk, ascending, src_result); + ASSERT_TRUE(status.ok()); + rc.RecordSection("topk"); + + CheckTopkResult(src_result, ascending, nq, topk); + rc.RecordSection("check result"); + }; + + DoTopk(5, 10, 4, false); + DoTopk(20005, 998, 123, true); + DoTopk(9987, 12, 10, false); + DoTopk(77777, 1000, 1, false); + DoTopk(5432, 8899, 8899, true); +} \ No newline at end of file From f986af86719ec427cd3909c2cac03d8e3f2dde3a Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 24 Jul 2019 14:09:06 +0800 Subject: [PATCH 18/24] MS-266 Improve topk reduce time by using multi-threads Former-commit-id: ec370a34b214c9105f06a9bdea8f34728ed5026a --- cpp/src/cache/CacheMgr.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/cache/CacheMgr.cpp b/cpp/src/cache/CacheMgr.cpp index 95deb4c80d..5e54c9abe1 100644 --- a/cpp/src/cache/CacheMgr.cpp +++ b/cpp/src/cache/CacheMgr.cpp @@ -49,7 +49,6 @@ DataObjPtr CacheMgr::GetItem(const std::string& key) { engine::Index_ptr CacheMgr::GetIndex(const std::string& key) { DataObjPtr obj = GetItem(key); if(obj != nullptr) { - SERVER_LOG_ERROR << "Can't get object from key: " << key; return obj->data(); } From 8b757539d2544d4efcdb0d9e154a131940e3b6a4 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 24 Jul 2019 16:28:47 +0800 Subject: [PATCH 19/24] MS-266 Improve topk reduce time by using multi-threads Former-commit-id: fb7b6588c31ca34b48e349a5b928bca2d21fbc20 --- cpp/src/db/DBImpl.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index ce3af88e2c..9a27f09b3d 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -89,7 +89,7 @@ DBImpl::DBImpl(const Options& options) meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode); mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_); if (options.mode != Options::MODE::READ_ONLY) { - ENGINE_LOG_INFO << "StartTimerTasks"; + ENGINE_LOG_TRACE << "StartTimerTasks"; StartTimerTasks(); } @@ -297,7 +297,7 @@ void DBImpl::StartMetricTask() { return; } - ENGINE_LOG_INFO << "Start metric task"; + ENGINE_LOG_TRACE << "Start metric task"; server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); @@ -312,7 +312,7 @@ void DBImpl::StartMetricTask() { server::Metrics::GetInstance().GPUMemoryUsageGaugeSet(); server::Metrics::GetInstance().OctetsSet(); - ENGINE_LOG_INFO << "Metric task finished"; + ENGINE_LOG_TRACE << "Metric task finished"; } void DBImpl::StartCompactionTask() { @@ -433,7 +433,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { } void DBImpl::BackgroundCompaction(std::set table_ids) { - ENGINE_LOG_INFO << " Background compaction thread start"; + ENGINE_LOG_TRACE << " Background compaction thread start"; Status status; for (auto& table_id : table_ids) { @@ -452,7 +452,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { } meta_ptr_->CleanUpFilesWithTTL(ttl); - ENGINE_LOG_INFO << " Background compaction thread exit"; + ENGINE_LOG_TRACE << " Background compaction thread exit"; } void DBImpl::StartBuildIndexTask(bool force) { @@ -581,7 +581,7 @@ Status DBImpl::BuildIndexByTable(const std::string& table_id) { } void DBImpl::BackgroundBuildIndex() { - ENGINE_LOG_INFO << " Background build index thread start"; + ENGINE_LOG_TRACE << " Background build index thread start"; std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; @@ -599,7 +599,7 @@ void DBImpl::BackgroundBuildIndex() { } } - ENGINE_LOG_INFO << " Background build index thread exit"; + ENGINE_LOG_TRACE << " Background build index thread exit"; } Status DBImpl::DropAll() { From a68e501b4b005b3695be764d7f047c097cd6f85d Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 24 Jul 2019 19:48:01 +0800 Subject: [PATCH 20/24] MS-266 Improve topk reduce time by using multi-threads Former-commit-id: f19c6d78f7f9ab619de792fbb43efd86da0b07a1 --- cpp/conf/server_config.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 8c81315de9..ff5cfc6cd5 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -8,7 +8,7 @@ db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path db_slave_path: # secondry data storage path, split by semicolon - parallel_reduce: true # use multi-threads to reduce topk result + parallel_reduce: false # use multi-threads to reduce topk result # URI format: dialect://username:password@host:port/database # All parts except dialect are optional, but you MUST include the delimiters From 6acfd7f60db6554504eaed817f3eafb912a00247 Mon Sep 17 00:00:00 2001 From: jinhai Date: Wed, 24 Jul 2019 19:59:19 +0800 Subject: [PATCH 21/24] MS-276 Fix compile with volta architecture Former-commit-id: 5d67c583a462764d368b3d83d337849d54c955f7 --- cpp/cmake/ThirdPartyPackages.cmake | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index fed5dece88..871c2ce828 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -752,10 +752,7 @@ macro(build_faiss) if(${MILVUS_WITH_FAISS_GPU_VERSION} STREQUAL "ON") set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" - "--with-cuda-arch=\"-gencode=arch=compute_35,code=sm_35\"" - "--with-cuda-arch=\"-gencode=arch=compute_52,code=sm_52\"" - "--with-cuda-arch=\"-gencode=arch=compute_60,code=sm_60\"" - "--with-cuda-arch=\"-gencode=arch=compute_61,code=sm_61\"" + "--with-cuda-arch=-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_52,code=sm_52 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_75,code=sm_75" ) else() set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda) @@ -769,7 +766,7 @@ macro(build_faiss) "./configure" ${FAISS_CONFIGURE_ARGS} BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} + ${MAKE} ${MAKE_BUILD_ARGS} VERBOSE=1 BUILD_IN_SOURCE 1 INSTALL_COMMAND From 6d27cc91bb4e3a2549f6e074473abacf2fba0b81 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 24 Jul 2019 20:14:05 +0800 Subject: [PATCH 22/24] MS-277 - Update CUDA Version to V10.1 Former-commit-id: ae02d3236cc34ce89f5e2a453e2c027c1562fc98 --- CHANGELOGS.md | 1 + ci/main_jenkinsfile | 2 +- ci/main_jenkinsfile_no_ut | 2 +- ci/nightly_main_jenkinsfile | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOGS.md b/CHANGELOGS.md index bbe0d9fc60..c8ee3b39fd 100644 --- a/CHANGELOGS.md +++ b/CHANGELOGS.md @@ -18,3 +18,4 @@ Please mark all change in change log and use the ticket from JIRA. - MS-161 - Add CI / CD Module to Milvus Project - MS-202 - Add Milvus Jenkins project email notification - MS-215 - Add Milvus cluster CI/CD groovy file +- MS-277 - Update CUDA Version to V10.1 diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index d4132e9ff1..2710e51ffb 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -35,7 +35,7 @@ pipeline { defaultContainer 'jnlp' containerTemplate { name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.11' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.12' ttyEnabled true command 'cat' } diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 0b2f90fd63..4b0fa812ff 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -35,7 +35,7 @@ pipeline { defaultContainer 'jnlp' containerTemplate { name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.11' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.12' ttyEnabled true command 'cat' } diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index 5458cf7632..567e70cb48 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -35,7 +35,7 @@ pipeline { defaultContainer 'jnlp' containerTemplate { name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.11' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.12' ttyEnabled true command 'cat' } From 837da04dc9af56129d5cf4cdc112111399730ea1 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 24 Jul 2019 20:34:11 +0800 Subject: [PATCH 23/24] add IndexStatsHelper Former-commit-id: 6579465eb992d10a9a2f10b0f129341664f9b77d --- cpp/src/db/FaissExecutionEngine.cpp | 73 +++++++++++++++++++---------- cpp/src/db/FaissExecutionEngine.h | 39 +++++++++++---- 2 files changed, 77 insertions(+), 35 deletions(-) diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index fc798afb9c..94f1299477 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -30,10 +30,31 @@ std::string GetMetricType() { } } +std::string IndexStatsHelper::ToString(const std::string &prefix) const { + return ""; +} + +void IndexStatsHelper::Reset() const { + faiss::indexIVF_stats.reset(); +} + +std::string FaissIndexIVFStatsHelper::ToString(const std::string &prefix) const { + std::stringstream ss; + ss << prefix; + ss << identifier_ << ":"; + ss << " NQ=" << faiss::indexIVF_stats.nq; + ss << " NL=" << faiss::indexIVF_stats.nlist; + ss << " ND=" << faiss::indexIVF_stats.ndis; + ss << " NH=" << faiss::indexIVF_stats.nheap_updates; + ss << " Q=" << faiss::indexIVF_stats.quantization_time; + ss << " S=" << faiss::indexIVF_stats.search_time; + return ss.str(); +} + FaissExecutionEngine::FaissExecutionEngine(uint16_t dimension, - const std::string& location, - const std::string& build_index_type, - const std::string& raw_index_type) + const std::string &location, + const std::string &build_index_type, + const std::string &raw_index_type) : location_(location), build_index_type_(build_index_type), raw_index_type_(raw_index_type) { @@ -44,9 +65,9 @@ FaissExecutionEngine::FaissExecutionEngine(uint16_t dimension, } FaissExecutionEngine::FaissExecutionEngine(std::shared_ptr index, - const std::string& location, - const std::string& build_index_type, - const std::string& raw_index_type) + const std::string &location, + const std::string &build_index_type, + const std::string &raw_index_type) : pIndex_(index), location_(location), build_index_type_(build_index_type), @@ -59,11 +80,11 @@ Status FaissExecutionEngine::AddWithIds(long n, const float *xdata, const long * } size_t FaissExecutionEngine::Count() const { - return (size_t)(pIndex_->ntotal); + return (size_t) (pIndex_->ntotal); } size_t FaissExecutionEngine::Size() const { - return (size_t)(Count() * pIndex_->d)*sizeof(float); + return (size_t) (Count() * pIndex_->d) * sizeof(float); } size_t FaissExecutionEngine::Dimension() const { @@ -80,7 +101,7 @@ Status FaissExecutionEngine::Serialize() { } Status FaissExecutionEngine::Load(bool to_cache) { - auto index = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location_); + auto index = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location_); bool already_in_cache = (index != nullptr); auto start_time = METRICS_NOW_TIME; if (!index) { @@ -99,12 +120,12 @@ Status FaissExecutionEngine::Load(bool to_cache) { server::Metrics::GetInstance().FaissDiskLoadSizeBytesHistogramObserve(total_size); // server::Metrics::GetInstance().FaissDiskLoadIOSpeedHistogramObserve(total_size/double(total_time)); - server::Metrics::GetInstance().FaissDiskLoadIOSpeedGaugeSet(total_size/double(total_time)); + server::Metrics::GetInstance().FaissDiskLoadIOSpeedGaugeSet(total_size / double(total_time)); } return Status::OK(); } -Status FaissExecutionEngine::Merge(const std::string& location) { +Status FaissExecutionEngine::Merge(const std::string &location) { if (location == location_) { return Status::Error("Cannot Merge Self"); } @@ -114,14 +135,14 @@ Status FaissExecutionEngine::Merge(const std::string& location) { if (!to_merge) { to_merge = read_index(location); } - auto file_index = dynamic_cast(to_merge->data().get()); - pIndex_->add_with_ids(file_index->ntotal, dynamic_cast(file_index->index)->xb.data(), - file_index->id_map.data()); + auto file_index = dynamic_cast(to_merge->data().get()); + pIndex_->add_with_ids(file_index->ntotal, dynamic_cast(file_index->index)->xb.data(), + file_index->id_map.data()); return Status::OK(); } ExecutionEnginePtr -FaissExecutionEngine::BuildIndex(const std::string& location) { +FaissExecutionEngine::BuildIndex(const std::string &location) { ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_; auto opd = std::make_shared(); @@ -130,11 +151,11 @@ FaissExecutionEngine::BuildIndex(const std::string& location) { opd->metric_type = GetMetricType(); IndexBuilderPtr pBuilder = GetIndexBuilder(opd); - auto from_index = dynamic_cast(pIndex_.get()); + auto from_index = dynamic_cast(pIndex_.get()); auto index = pBuilder->build_all(from_index->ntotal, - dynamic_cast(from_index->index)->xb.data(), - from_index->id_map.data()); + dynamic_cast(from_index->index)->xb.data(), + from_index->id_map.data()); ExecutionEnginePtr new_ee(new FaissExecutionEngine(index->data(), location, build_index_type_, raw_index_type_)); return new_ee; @@ -148,18 +169,21 @@ Status FaissExecutionEngine::Search(long n, auto start_time = METRICS_NOW_TIME; std::shared_ptr ivf_index = std::dynamic_pointer_cast(pIndex_); - if(ivf_index) { + if (ivf_index) { + std::string stats_prefix = "K=" + std::to_string(k) + ":"; ENGINE_LOG_DEBUG << "Searching index type: " << build_index_type_ << " nProbe: " << nprobe_; ivf_index->nprobe = nprobe_; + ivf_stats_helper_.Reset(); ivf_index->search(n, data, k, distances, labels); + ENGINE_LOG_INFO << ivf_stats_helper_.ToString(stats_prefix); } else { ENGINE_LOG_DEBUG << "Searching raw file"; pIndex_->search(n, data, k, distances, labels); } auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time,end_time); - server::Metrics::GetInstance().QueryIndexTypePerSecondSet(build_index_type_, double(n)/double(total_time)); + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().QueryIndexTypePerSecondSet(build_index_type_, double(n) / double(total_time)); return Status::OK(); } @@ -173,17 +197,16 @@ Status FaissExecutionEngine::Cache() { Status FaissExecutionEngine::Init() { - if(build_index_type_ == BUILD_INDEX_TYPE_IVF || + if (build_index_type_ == BUILD_INDEX_TYPE_IVF || build_index_type_ == BUILD_INDEX_TYPE_IVFSQ8) { using namespace zilliz::milvus::server; ServerConfig &config = ServerConfig::GetInstance(); ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000); - nlist_ = engine_config.GetInt32Value(CONFIG_NLIST,16384); + nlist_ = engine_config.GetInt32Value(CONFIG_NLIST, 16384); - } else if(build_index_type_ == BUILD_INDEX_TYPE_IDMAP) { - ; + } else if (build_index_type_ == BUILD_INDEX_TYPE_IDMAP) { ; } else { return Status::Error("Wrong index type: ", build_index_type_); } diff --git a/cpp/src/db/FaissExecutionEngine.h b/cpp/src/db/FaissExecutionEngine.h index c1baf48be3..512743ca36 100644 --- a/cpp/src/db/FaissExecutionEngine.h +++ b/cpp/src/db/FaissExecutionEngine.h @@ -11,6 +11,7 @@ #include #include + namespace zilliz { namespace milvus { namespace engine { @@ -19,18 +20,35 @@ const static std::string BUILD_INDEX_TYPE_IDMAP = "IDMap"; const static std::string BUILD_INDEX_TYPE_IVF = "IVF"; const static std::string BUILD_INDEX_TYPE_IVFSQ8 = "IVFSQ8"; +class IndexStatsHelper { + + public: + using Ptr = std::shared_ptr; + virtual std::string ToString(const std::string &prefix = "") const; + virtual void Reset() const; + virtual ~IndexStatsHelper() {} +}; + +class FaissIndexIVFStatsHelper : public IndexStatsHelper { + public: + std::string ToString(const std::string &prefix = "") const override; + + private: + const std::string identifier_ = BUILD_INDEX_TYPE_IVF; +}; + class FaissExecutionEngine : public ExecutionEngine { -public: + public: FaissExecutionEngine(uint16_t dimension, - const std::string& location, - const std::string& build_index_type, - const std::string& raw_index_type); + const std::string &location, + const std::string &build_index_type, + const std::string &raw_index_type); FaissExecutionEngine(std::shared_ptr index, - const std::string& location, - const std::string& build_index_type, - const std::string& raw_index_type); + const std::string &location, + const std::string &build_index_type, + const std::string &raw_index_type); Status AddWithIds(long n, const float *xdata, const long *xids) override; @@ -46,7 +64,7 @@ public: Status Load(bool to_cache) override; - Status Merge(const std::string& location) override; + Status Merge(const std::string &location) override; Status Search(long n, const float *data, @@ -54,13 +72,14 @@ public: float *distances, long *labels) const override; - ExecutionEnginePtr BuildIndex(const std::string&) override; + ExecutionEnginePtr BuildIndex(const std::string &) override; Status Cache() override; Status Init() override; -protected: + protected: + FaissIndexIVFStatsHelper ivf_stats_helper_; std::shared_ptr pIndex_; std::string location_; From 99f92d8758702420a746a31f9f34d574741eb6d7 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 24 Jul 2019 20:36:35 +0800 Subject: [PATCH 24/24] add IndexStatsHelper Former-commit-id: d47bb68ff89728b31d237a62caa4d4d7673ceb0f --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index b326293e27..642921e1fc 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -37,6 +37,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-260 - Refine log - MS-249 - Check machine hardware during initialize - MS-261 - Update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option +- MS-278 - add IndexStatsHelper ## New Feature - MS-180 - Add new mem manager