diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e62b98b50..52041d7e30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,31 +4,23 @@ Please mark all change in change log and use the ticket from JIRA. # Milvus 0.6.0 (TODO) ## Bug - +- \#228 - memory usage increased slowly during searching vectors - \#246 - Exclude src/external folder from code coverage for jenkin ci - \#248 - Reside src/external in thirdparty -## - ## Feature - - \#12 - Pure CPU version for Milvus - \#77 - Support table partition - \#226 - Experimental shards middleware for Milvus -- #127 - Support new Index type IVFPQ - -## +- \#127 - Support new Index type IVFPQ ## Improvement - - \#275 - Rename C++ SDK IndexType - \#284 - Change C++ SDK to shared library - \#260 - C++ SDK README - \#314 - add Find FAISS in CMake - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue -## - ## Task # Milvus 0.5.3 (2019-11-13) @@ -117,7 +109,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-658 - Fix SQ8 Hybrid can't search - MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources - \#9 - Change default gpu_cache_capacity to 4 -- \#20 - C++ sdk example get grpc error +- \#20 - C++ sdk example get grpc error - \#23 - Add unittest to improve code coverage - \#31 - make clang-format failed after run build.sh -l - \#39 - Create SQ8H index hang if using github server version @@ -169,7 +161,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-635 - Add compile option to support customized faiss - MS-660 - add ubuntu_build_deps.sh - \#18 - Add all test cases - + # Milvus 0.4.0 (2019-09-12) ## Bug @@ -378,11 +370,11 @@ Please mark all change in change log and use the ticket from JIRA. - MS-82 - Update server startup welcome message - MS-83 - Update vecwise to Milvus - MS-77 - Performance issue of post-search action -- MS-22 - Enhancement for MemVector size control +- MS-22 - Enhancement for MemVector size control - MS-92 - Unify behavior of debug and release build - MS-98 - Install all unit test to installation directory - MS-115 - Change is_startup of metric_config switch from true to on -- MS-122 - Archive criteria config +- MS-122 - Archive criteria config - MS-124 - HasTable interface - MS-126 - Add more error code - MS-128 - Change default db path diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 45a2bc5fe6..f24935f788 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -33,7 +33,7 @@ message(STATUS "Build time = ${BUILD_TIME}") MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'" - OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) ENDMACRO(GET_GIT_BRANCH_NAME) GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) @@ -117,17 +117,17 @@ include(DefineOptions) include(BuildUtils) include(ThirdPartyPackages) -if(MILVUS_USE_CCACHE) - find_program(CCACHE_FOUND ccache) - if(CCACHE_FOUND) - message(STATUS "Using ccache: ${CCACHE_FOUND}") - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) - set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) - # let ccache preserve C++ comments, because some of them may be - # meaningful to the compiler - set(ENV{CCACHE_COMMENTS} "1") - endif(CCACHE_FOUND) -endif() +if (MILVUS_USE_CCACHE) + find_program(CCACHE_FOUND ccache) + if (CCACHE_FOUND) + message(STATUS "Using ccache: ${CCACHE_FOUND}") + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) + # let ccache preserve C++ comments, because some of them may be + # meaningful to the compiler + set(ENV{CCACHE_COMMENTS} "1") + endif (CCACHE_FOUND) +endif () set(MILVUS_CPU_VERSION false) if (MILVUS_GPU_VERSION) @@ -142,6 +142,10 @@ else () add_compile_definitions("MILVUS_CPU_VERSION") endif () +if (MILVUS_WITH_PROMETHEUS) + add_compile_definitions("MILVUS_WITH_PROMETHEUS") +endif () + if (CMAKE_BUILD_TYPE STREQUAL "Release") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") if (MILVUS_GPU_VERSION) @@ -176,9 +180,9 @@ endif () if (MILVUS_GPU_VERSION) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_gpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) -else() +else () configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_cpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) -endif() +endif () configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) diff --git a/core/build.sh b/core/build.sh index 9b690a0261..85dea0ab88 100755 --- a/core/build.sh +++ b/core/build.sh @@ -14,10 +14,11 @@ CUSTOMIZATION="OFF" # default use ori faiss CUDA_COMPILER=/usr/local/cuda/bin/nvcc GPU_VERSION="OFF" #defaults to CPU version WITH_MKL="OFF" -FAISS_ROOT="" +FAISS_ROOT="" #FAISS root path FAISS_SOURCE="BUNDLED" +WITH_PROMETHEUS="ON" -while getopts "p:d:t:f:ulrcgjhxzm" arg +while getopts "p:d:t:f:ulrcgjhxzme" arg do case $arg in p) @@ -63,7 +64,10 @@ do ;; m) WITH_MKL="ON" - ;; + ;; + e) + WITH_PROMETHEUS="OFF" + ;; h) # help echo " @@ -80,10 +84,11 @@ parameter: -j: use jfrog cache build directory(default: OFF) -g: build GPU version(default: OFF) -m: build with MKL(default: OFF) +-e: build without prometheus -h: help usage: -./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h] +./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-e] [-h] " exit 0 ;; @@ -118,6 +123,7 @@ CMAKE_CMD="cmake \ -DCUSTOMIZATION=${CUSTOMIZATION} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ -DFAISS_WITH_MKL=${WITH_MKL} \ +-DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \ ../" echo ${CMAKE_CMD} ${CMAKE_CMD} diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 406518f1fb..154db5d134 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -29,7 +29,7 @@ metric_config: cache_config: cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] - gpu_cache_capacity: 3 # GB, GPU memory used for cache, must be a positive integer + gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] cache_insert_data: false # whether to load inserted data into cache, must be a boolean diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 937c0716a4..aa79c084f1 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -37,6 +37,7 @@ endforeach () aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/metrics/prometheus metrics_prometheus_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) @@ -91,6 +92,11 @@ set(engine_files ${wrapper_files} ) +if (MILVUS_WITH_PROMETHEUS) + set(engine_files ${engine_files} + ${metrics_prometheus_files}) +endif () + set(client_grpc_lib grpcpp_channelz grpc++ @@ -115,7 +121,6 @@ set(third_party_libs sqlite ${client_grpc_lib} yaml-cpp - ${prometheus_lib} mysqlpp zlib ${boost_lib} @@ -138,13 +143,19 @@ if (MILVUS_GPU_VERSION) ) endif () -if (MILVUS_ENABLE_PROFILING STREQUAL "ON") +if (MILVUS_ENABLE_PROFILING) set(third_party_libs ${third_party_libs} gperftools libunwind ) endif () +if (MILVUS_WITH_PROMETHEUS) + set(third_party_libs ${third_party_libs} + ${prometheus_lib} + ) +endif () + set(engine_libs pthread libgomp.a @@ -166,13 +177,22 @@ target_link_libraries(milvus_engine ${engine_libs} ) -add_library(metrics STATIC ${metrics_files}) +if (MILVUS_WITH_PROMETHEUS) + add_library(metrics STATIC ${metrics_files} ${metrics_prometheus_files}) +else () + add_library(metrics STATIC ${metrics_files}) +endif () set(metrics_lib yaml-cpp - ${prometheus_lib} ) +if (MILVUS_WITH_PROMETHEUS) + set(metrics_lib ${metrics_lib} + ${prometheus_lib} + ) +endif () + target_link_libraries(metrics ${metrics_lib}) set(server_libs diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 3e0501b84e..d688c88ff2 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -619,6 +619,18 @@ DBImpl::StartCompactionTask() { { std::lock_guard lck(compact_result_mutex_); if (compact_thread_results_.empty()) { + // collect merge files for all tables(if compact_table_ids_ is empty) for two reasons: + // 1. other tables may still has un-merged files + // 2. server may be closed unexpected, these un-merge files need to be merged when server restart + if (compact_table_ids_.empty()) { + std::vector table_schema_array; + meta_ptr_->AllTables(table_schema_array); + for (auto& schema : table_schema_array) { + compact_table_ids_.insert(schema.table_id_); + } + } + + // start merge file thread compact_thread_results_.push_back( compact_thread_pool_.enqueue(&DBImpl::BackgroundCompaction, this, compact_table_ids_)); compact_table_ids_.clear(); @@ -717,7 +729,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) { for (auto& kv : raw_files) { auto files = kv.second; if (files.size() < options_.merge_trigger_number_) { - ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action"; + ENGINE_LOG_TRACE << "Files number not greater equal than merge trigger number, skip merge action"; continue; } @@ -734,7 +746,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) { void DBImpl::BackgroundCompaction(std::set table_ids) { - ENGINE_LOG_TRACE << " Background compaction thread start"; + ENGINE_LOG_TRACE << "Background compaction thread start"; Status status; for (auto& table_id : table_ids) { @@ -757,7 +769,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { } meta_ptr_->CleanUpFilesWithTTL(ttl); - ENGINE_LOG_TRACE << " Background compaction thread exit"; + ENGINE_LOG_TRACE << "Background compaction thread exit"; } void diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index ff36554c10..bf83447806 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1392,6 +1392,7 @@ MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFiles } // Scoped Connection Status ret; + int64_t to_merge_files = 0; for (auto& resRow : res) { TableFileSchema table_file; table_file.file_size_ = resRow["file_size"]; @@ -1420,13 +1421,14 @@ MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFiles auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { files[table_file.date_] = TableFilesSchema(); + to_merge_files++; } files[table_file.date_].push_back(table_file); } - if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-merge files"; + if (to_merge_files > 0) { + ENGINE_LOG_TRACE << "Collect " << to_merge_files << " to-merge files"; } return ret; } catch (std::exception& e) { @@ -1809,6 +1811,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + int64_t remove_tables = 0; if (!res.empty()) { std::stringstream idsToDeleteSS; for (auto& resRow : res) { @@ -1817,7 +1820,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { resRow["table_id"].to_string(table_id); utils::DeleteTablePath(options_, table_id, false); // only delete empty folder - + remove_tables++; idsToDeleteSS << "id = " << std::to_string(id) << " OR "; } std::string idsToDeleteStr = idsToDeleteSS.str(); @@ -1832,8 +1835,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } } - if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Remove " << res.size() << " tables from meta"; + if (remove_tables > 0) { + ENGINE_LOG_DEBUG << "Remove " << remove_tables << " tables from meta"; } } // Scoped Connection } catch (std::exception& e) { diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 6221dd8ac1..22e953fe9d 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -971,6 +971,7 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile order_by(&TableFileSchema::file_size_).desc()); Status result; + int64_t to_merge_files = 0; for (auto& file : selected) { TableFileSchema table_file; table_file.file_size_ = std::get<4>(file); @@ -999,11 +1000,13 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile if (dateItr == files.end()) { files[table_file.date_] = TableFilesSchema(); } + files[table_file.date_].push_back(table_file); + to_merge_files++; } - if (selected.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-merge files"; + if (to_merge_files > 0) { + ENGINE_LOG_TRACE << "Collect " << to_merge_files << " to-merge files"; } return result; } catch (std::exception& e) { @@ -1313,16 +1316,18 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { try { server::MetricCollector metric; + int64_t remove_tables = 0; for (auto& table_id : table_ids) { auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_), where(c(&TableFileSchema::table_id_) == table_id)); if (selected.size() == 0) { utils::DeleteTablePath(options_, table_id); + remove_tables++; } } - if (table_ids.size() > 0) { - ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; + if (remove_tables) { + ENGINE_LOG_DEBUG << "Remove " << remove_tables << " tables folder"; } } catch (std::exception& e) { return HandleException("Encounter exception when delete table folder", e.what()); diff --git a/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp b/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp index b4c3910a01..db4a415261 100644 --- a/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp +++ b/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp @@ -89,34 +89,35 @@ ConvertToDataset(std::vector query_results) { } } - auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems); - - // TODO: magic - std::vector id_bufs{nullptr, id_buf}; - std::vector dist_bufs{nullptr, dist_buf}; - - auto int64_type = std::make_shared(); - auto float_type = std::make_shared(); - - auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); - auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); - // auto id_array_data = std::make_shared(int64_type, sizeof(int64_t) * elems, id_bufs); - // auto dist_array_data = std::make_shared(float_type, sizeof(float) * elems, dist_bufs); - - // auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems); - // auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems); - - auto ids = std::make_shared>(id_array_data); - auto dists = std::make_shared>(dist_array_data); - std::vector array{ids, dists}; - - auto field_id = std::make_shared("id", std::make_shared()); - auto field_dist = std::make_shared("dist", std::make_shared()); - std::vector fields{field_id, field_dist}; - auto schema = std::make_shared(fields); - - return std::make_shared(array, schema); + // auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems); + // auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems); + // + // // TODO: magic + // std::vector id_bufs{nullptr, id_buf}; + // std::vector dist_bufs{nullptr, dist_buf}; + // + // auto int64_type = std::make_shared(); + // auto float_type = std::make_shared(); + // + // auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); + // auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); + // // auto id_array_data = std::make_shared(int64_type, sizeof(int64_t) * elems, id_bufs); + // // auto dist_array_data = std::make_shared(float_type, sizeof(float) * elems, dist_bufs); + // + // // auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems); + // // auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems); + // + // auto ids = std::make_shared>(id_array_data); + // auto dists = std::make_shared>(dist_array_data); + // std::vector array{ids, dists}; + // + // auto field_id = std::make_shared("id", std::make_shared()); + // auto field_dist = std::make_shared("dist", std::make_shared()); + // std::vector fields{field_id, field_dist}; + // auto schema = std::make_shared(fields); + // + // return std::make_shared(array, schema); + return std::make_shared((void*)p_id, (void*)p_dist); } } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/common/Dataset.h b/core/src/index/knowhere/knowhere/common/Dataset.h index 1331239dd6..b101aba6a7 100644 --- a/core/src/index/knowhere/knowhere/common/Dataset.h +++ b/core/src/index/knowhere/knowhere/common/Dataset.h @@ -54,6 +54,9 @@ class Dataset { : tensor_(std::move(tensor)), tensor_schema_(std::move(tensor_schema)) { } + Dataset(void* ids, void* dists) : ids_(ids), dists_(dists) { + } + Dataset(const Dataset&) = delete; Dataset& operator=(const Dataset&) = delete; @@ -128,6 +131,16 @@ class Dataset { tensor_schema_ = std::move(tensor_schema); } + void* + ids() { + return ids_; + } + + void* + dist() { + return dists_; + } + // const Config & // meta() const { return meta_; } @@ -141,6 +154,9 @@ class Dataset { SchemaPtr array_schema_; std::vector tensor_; SchemaPtr tensor_schema_; + // TODO(yukun): using smart pointer + void* ids_; + void* dists_; // Config meta_; }; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index 98d25e5e5c..7aedf98613 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -80,23 +80,24 @@ IDMAP::Search(const DatasetPtr& dataset, const Config& config) { search_impl(rows, (float*)p_data, config->k, res_dis, res_ids, Config()); - auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); - - std::vector id_bufs{nullptr, id_buf}; - std::vector dist_bufs{nullptr, dist_buf}; - - auto int64_type = std::make_shared(); - auto float_type = std::make_shared(); - - auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); - auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); - - auto ids = std::make_shared>(id_array_data); - auto dists = std::make_shared>(dist_array_data); - std::vector array{ids, dists}; - - return std::make_shared(array, nullptr); + // auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); + // auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); + // + // std::vector id_bufs{nullptr, id_buf}; + // std::vector dist_bufs{nullptr, dist_buf}; + // + // auto int64_type = std::make_shared(); + // auto float_type = std::make_shared(); + // + // auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); + // auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); + // + // auto ids = std::make_shared>(id_array_data); + // auto dists = std::make_shared>(dist_array_data); + // std::vector array{ids, dists}; + // + // return std::make_shared(array, nullptr); + return std::make_shared((void*)res_ids, (void*)res_dis); } void diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index b2a2af29a8..7f30a97ea0 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -139,23 +139,23 @@ IVF::Search(const DatasetPtr& dataset, const Config& config) { // std::cout << ss_res_id.str() << std::endl; // std::cout << ss_res_dist.str() << std::endl << std::endl; - auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); + // auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); + // auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); + // + // std::vector id_bufs{nullptr, id_buf}; + // std::vector dist_bufs{nullptr, dist_buf}; + // + // auto int64_type = std::make_shared(); + // auto float_type = std::make_shared(); + // + // auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); + // auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); + // + // auto ids = std::make_shared>(id_array_data); + // auto dists = std::make_shared>(dist_array_data); + // std::vector array{ids, dists}; - std::vector id_bufs{nullptr, id_buf}; - std::vector dist_bufs{nullptr, dist_buf}; - - auto int64_type = std::make_shared(); - auto float_type = std::make_shared(); - - auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); - auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); - - auto ids = std::make_shared>(id_array_data); - auto dists = std::make_shared>(dist_array_data); - std::vector array{ids, dists}; - - return std::make_shared(array, nullptr); + return std::make_shared((void*)res_ids, (void*)res_dis); } void diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp index 8f6d93d7ff..204819517a 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -88,23 +88,24 @@ NSG::Search(const DatasetPtr& dataset, const Config& config) { s_params.search_length = build_cfg->search_length; index_->Search((float*)p_data, rows, dim, build_cfg->k, res_dis, res_ids, s_params); - auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); + // auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); + // auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); - std::vector id_bufs{nullptr, id_buf}; - std::vector dist_bufs{nullptr, dist_buf}; - - auto int64_type = std::make_shared(); - auto float_type = std::make_shared(); - - auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); - auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); - - auto ids = std::make_shared>(id_array_data); - auto dists = std::make_shared>(dist_array_data); - std::vector array{ids, dists}; - - return std::make_shared(array, nullptr); + // std::vector id_bufs{nullptr, id_buf}; + // std::vector dist_bufs{nullptr, dist_buf}; + // + // auto int64_type = std::make_shared(); + // auto float_type = std::make_shared(); + // + // auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); + // auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); + // + // auto ids = std::make_shared>(id_array_data); + // auto dists = std::make_shared>(dist_array_data); + // std::vector array{ids, dists}; + // + // return std::make_shared(array, nullptr); + return std::make_shared((void*)res_ids, (void*)res_dis); } IndexModelPtr diff --git a/core/src/index/unittest/test_ivf.cpp b/core/src/index/unittest/test_ivf.cpp index fa294adc0e..79ef1aa170 100644 --- a/core/src/index/unittest/test_ivf.cpp +++ b/core/src/index/unittest/test_ivf.cpp @@ -181,11 +181,13 @@ TEST_P(IVFTest, clone_test) { // PrintResult(result, nq, k); auto AssertEqual = [&](knowhere::DatasetPtr p1, knowhere::DatasetPtr p2) { - auto ids_p1 = p1->array()[0]; - auto ids_p2 = p2->array()[0]; + auto ids_p1 = p1->ids(); + auto ids_p2 = p2->ids(); for (int i = 0; i < nq * k; ++i) { - EXPECT_EQ(*(ids_p2->data()->GetValues(1, i)), *(ids_p1->data()->GetValues(1, i))); + EXPECT_EQ(*((int64_t*)(ids_p2) + i), *((int64_t*)(ids_p1) + i)); + // EXPECT_EQ(*(ids_p2->data()->GetValues(1, i)), *(ids_p1->data()->GetValues(1, + // i))); } }; diff --git a/core/src/index/unittest/test_kdt.cpp b/core/src/index/unittest/test_kdt.cpp index 5400881875..bbc7dcf94c 100644 --- a/core/src/index/unittest/test_kdt.cpp +++ b/core/src/index/unittest/test_kdt.cpp @@ -66,15 +66,19 @@ TEST_F(KDTTest, kdt_basic) { AssertAnns(result, nq, k); { - auto ids = result->array()[0]; - auto dists = result->array()[1]; + // auto ids = result->array()[0]; + // auto dists = result->array()[1]; + auto ids = result->ids(); + auto dists = result->dist(); std::stringstream ss_id; std::stringstream ss_dist; for (auto i = 0; i < nq; i++) { for (auto j = 0; j < k; ++j) { - ss_id << *ids->data()->GetValues(1, i * k + j) << " "; - ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; + ss_id << *((int64_t*)(ids) + i * k + j) << " "; + ss_dist << *((float*)(dists) + i * k + j) << " "; + // ss_id << *ids->data()->GetValues(1, i * k + j) << " "; + // ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; } ss_id << std::endl; ss_dist << std::endl; diff --git a/core/src/index/unittest/utils.cpp b/core/src/index/unittest/utils.cpp index d4a59bafbb..2556b60fad 100644 --- a/core/src/index/unittest/utils.cpp +++ b/core/src/index/unittest/utils.cpp @@ -151,9 +151,10 @@ generate_query_dataset(int64_t nb, int64_t dim, float* xb) { void AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { - auto ids = result->array()[0]; + auto ids = result->ids(); for (auto i = 0; i < nq; i++) { - EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); + EXPECT_EQ(i, *((int64_t*)(ids) + i * k)); + // EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); } } diff --git a/core/src/metrics/Metrics.cpp b/core/src/metrics/Metrics.cpp index 51db5555b8..5fd3553cdc 100644 --- a/core/src/metrics/Metrics.cpp +++ b/core/src/metrics/Metrics.cpp @@ -16,8 +16,10 @@ // under the License. #include "metrics/Metrics.h" -#include "PrometheusMetrics.h" #include "server/Config.h" +#ifdef MILVUS_WITH_PROMETHEUS +#include "metrics/prometheus/PrometheusMetrics.h" +#endif #include @@ -37,11 +39,15 @@ Metrics::CreateMetricsCollector() { config.GetMetricConfigCollector(collector_type_str); +#ifdef MILVUS_WITH_PROMETHEUS if (collector_type_str == "prometheus") { return PrometheusMetrics::GetInstance(); } else { return MetricsBase::GetInstance(); } +#else + return MetricsBase::GetInstance(); +#endif } } // namespace server diff --git a/core/src/metrics/PrometheusMetrics.cpp b/core/src/metrics/prometheus/PrometheusMetrics.cpp similarity index 98% rename from core/src/metrics/PrometheusMetrics.cpp rename to core/src/metrics/prometheus/PrometheusMetrics.cpp index 770b34dc47..19b2683280 100644 --- a/core/src/metrics/PrometheusMetrics.cpp +++ b/core/src/metrics/prometheus/PrometheusMetrics.cpp @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "metrics/PrometheusMetrics.h" -#include "SystemInfo.h" +#include "metrics/prometheus/PrometheusMetrics.h" #include "cache/GpuCacheMgr.h" +#include "metrics/SystemInfo.h" #include "server/Config.h" #include "utils/Log.h" diff --git a/core/src/metrics/PrometheusMetrics.h b/core/src/metrics/prometheus/PrometheusMetrics.h similarity index 99% rename from core/src/metrics/PrometheusMetrics.h rename to core/src/metrics/prometheus/PrometheusMetrics.h index ef60f9a231..5a452ca02c 100644 --- a/core/src/metrics/PrometheusMetrics.h +++ b/core/src/metrics/prometheus/PrometheusMetrics.h @@ -24,7 +24,7 @@ #include #include -#include "MetricBase.h" +#include "metrics/MetricBase.h" #include "utils/Error.h" #define METRICS_NOW_TIME std::chrono::system_clock::now() diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index 05293b53f1..e7967cbf59 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -84,8 +84,8 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i Config search_cfg = cfg; auto res = index_->Search(dataset, search_cfg); - auto ids_array = res->array()[0]; - auto dis_array = res->array()[1]; + // auto ids_array = res->array()[0]; + // auto dis_array = res->array()[1]; //{ // auto& ids = ids_array; @@ -104,12 +104,14 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i // std::cout << "dist\n" << ss_dist.str() << std::endl; //} - auto p_ids = ids_array->data()->GetValues(1, 0); - auto p_dist = dis_array->data()->GetValues(1, 0); + // auto p_ids = ids_array->data()->GetValues(1, 0); + // auto p_dist = dis_array->data()->GetValues(1, 0); // TODO(linxj): avoid copy here. - memcpy(ids, p_ids, sizeof(int64_t) * nq * k); - memcpy(dist, p_dist, sizeof(float) * nq * k); + memcpy(ids, res->ids(), sizeof(int64_t) * nq * k); + memcpy(dist, res->dist(), sizeof(float) * nq * k); + free(res->ids()); + free(res->dist()); } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 10ab362e77..01e1054f7e 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -110,12 +110,18 @@ set(unittest_libs pthread metrics gfortran - prometheus-cpp-pull - prometheus-cpp-push - prometheus-cpp-core - dl - z ) +if (MILVUS_WITH_PROMETHEUS) + set(unittest_libs ${unittest_libs} + prometheus-cpp-push + prometheus-cpp-pull + prometheus-cpp-core + ) +endif () +set(unittest_libs ${unittest_libs} + dl + z + ) if (MILVUS_GPU_VERSION) include_directories("${CUDA_INCLUDE_DIRS}") @@ -135,4 +141,4 @@ add_subdirectory(db) add_subdirectory(wrapper) add_subdirectory(metrics) add_subdirectory(scheduler) -add_subdirectory(server) \ No newline at end of file +add_subdirectory(server) diff --git a/core/unittest/metrics/CMakeLists.txt b/core/unittest/metrics/CMakeLists.txt index ad9d4e3943..11c25a71e4 100644 --- a/core/unittest/metrics/CMakeLists.txt +++ b/core/unittest/metrics/CMakeLists.txt @@ -18,10 +18,15 @@ #------------------------------------------------------------------------------- set(test_files - ${CMAKE_CURRENT_SOURCE_DIR}/test_metricbase.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/test_metrics.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/test_prometheus.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/utils.cpp) + test_metricbase.cpp + test_metrics.cpp + utils.cpp + ) + +if (MILVUS_WITH_PROMETHEUS) + set(test_files ${test_files} + test_prometheus.cpp) +endif () add_executable(test_metrics ${common_files} diff --git a/core/unittest/metrics/test_prometheus.cpp b/core/unittest/metrics/test_prometheus.cpp index 50e845d1c2..6e339b73b4 100644 --- a/core/unittest/metrics/test_prometheus.cpp +++ b/core/unittest/metrics/test_prometheus.cpp @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include "metrics/PrometheusMetrics.h" #include "server/Config.h" +#include "metrics/prometheus/PrometheusMetrics.h" #include #include diff --git a/core/unittest/wrapper/utils.h b/core/unittest/wrapper/utils.h index 917c130273..1eaa7ae4ec 100644 --- a/core/unittest/wrapper/utils.h +++ b/core/unittest/wrapper/utils.h @@ -24,12 +24,13 @@ #include #include #include -#include + #include "wrapper/VecIndex.h" #include "wrapper/utils.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" #include "wrapper/ConfAdapterMgr.h" +#include "wrapper/ConfAdapter.h" class DataGenBase; diff --git a/tests/milvus-java-test/pom.xml b/tests/milvus-java-test/pom.xml index 4da715e292..bff6f1de61 100644 --- a/tests/milvus-java-test/pom.xml +++ b/tests/milvus-java-test/pom.xml @@ -99,7 +99,7 @@ io.milvus milvus-sdk-java - 0.2.0-SNAPSHOT + 0.3.0 @@ -134,4 +134,4 @@ - \ No newline at end of file + diff --git a/tests/milvus_python_test/test_ping.py b/tests/milvus_python_test/test_ping.py index d63ab93f11..3831dcd6ac 100644 --- a/tests/milvus_python_test/test_ping.py +++ b/tests/milvus_python_test/test_ping.py @@ -1,7 +1,7 @@ import logging import pytest -__version__ = '0.5.1' +__version__ = '0.6.0' class TestPing: