From de35ab27fb4f4c8fbe3d6733ccc12d720aa04f66 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 25 Nov 2019 10:57:24 +0800 Subject: [PATCH 01/18] Remove unittest file from docker image --- ci/jenkins/step/package.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/jenkins/step/package.groovy b/ci/jenkins/step/package.groovy index f0444084f8..35d2d8cfdf 100644 --- a/ci/jenkins/step/package.groovy +++ b/ci/jenkins/step/package.groovy @@ -4,6 +4,7 @@ timeout(time: 5, unit: 'MINUTES') { sh "./yaml_processor.py merge -f /opt/milvus/conf/server_config.yaml -m ../yaml/update_server_config.yaml -i && rm /opt/milvus/conf/server_config.yaml.bak" sh "sed -i 's/\\/tmp\\/milvus/\\/opt\\/milvus/g' /opt/milvus/conf/log_config.conf" } + sh "rm -rf /opt/milvus/unittest" sh "tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz -C /opt/ milvus" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz") From d2c2f99f7233cfe00b6147aac99772a7eb29a9d2 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 25 Nov 2019 11:20:24 +0800 Subject: [PATCH 02/18] add badge in README.md --- README.md | 2 ++ README_CN.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 4a1151dbf8..3cb1b34c25 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ ![Release](https://img.shields.io/badge/release-v0.5.1-yellowgreen) ![Release_date](https://img.shields.io/badge/release%20date-November-yellowgreen) +[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus) + [中文版](README_CN.md) ## What is Milvus diff --git a/README_CN.md b/README_CN.md index 8f4d4b05aa..5daef16d24 100644 --- a/README_CN.md +++ b/README_CN.md @@ -7,6 +7,8 @@ ![Release](https://img.shields.io/badge/release-v0.5.0-orange) ![Release_date](https://img.shields.io/badge/release_date-October-yellowgreen) +[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus) + - [Slack 频道](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk) - [Twitter](https://twitter.com/milvusio) - [Facebook](https://www.facebook.com/io.milvus.5) From a3a8e222aa10f80018960c9311070330ccd4aad7 Mon Sep 17 00:00:00 2001 From: zhenwu Date: Mon, 25 Nov 2019 14:53:23 +0800 Subject: [PATCH 03/18] remove invalid params --- tests/milvus_python_test/test_index.py | 10 +++++----- tests/milvus_python_test/utils.py | 22 ++++++++++------------ 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/tests/milvus_python_test/test_index.py b/tests/milvus_python_test/test_index.py index 39aadb9d33..b253cf02a3 100644 --- a/tests/milvus_python_test/test_index.py +++ b/tests/milvus_python_test/test_index.py @@ -1208,9 +1208,9 @@ class TestCreateIndexParamsInvalid(object): nlist = index_params["nlist"] logging.getLogger().info(index_params) status, ids = connect.add_vectors(table, vectors) - # if not isinstance(index_type, int) or not isinstance(nlist, int): - with pytest.raises(Exception) as e: + if (not index_type) or (not nlist) or (not isinstance(index_type, IndexType)) or (not isinstance(nlist, int)): + with pytest.raises(Exception) as e: + status = connect.create_index(table, index_params) + else: status = connect.create_index(table, index_params) - # else: - # status = connect.create_index(table, index_params) - # assert not status.OK() + assert not status.OK() diff --git a/tests/milvus_python_test/utils.py b/tests/milvus_python_test/utils.py index 159c8407c6..1686ad7129 100644 --- a/tests/milvus_python_test/utils.py +++ b/tests/milvus_python_test/utils.py @@ -54,12 +54,12 @@ def gen_long_str(num): def gen_invalid_ips(): ips = [ - "255.0.0.0", - "255.255.0.0", - "255.255.255.0", - "255.255.255.255", + # "255.0.0.0", + # "255.255.0.0", + # "255.255.255.0", + # "255.255.255.255", "127.0.0", - "123.0.0.2", + # "123.0.0.2", "12-s", " ", "12 s", @@ -114,12 +114,12 @@ def gen_invalid_uris(): # invalid ip "tcp:// :%s" % port, - "tcp://123.0.0.1:%s" % port, + # "tcp://123.0.0.1:%s" % port, "tcp://127.0.0:%s" % port, "tcp://255.0.0.0:%s" % port, - "tcp://255.255.0.0:%s" % port, - "tcp://255.255.255.0:%s" % port, - "tcp://255.255.255.255:%s" % port, + # "tcp://255.255.0.0:%s" % port, + # "tcp://255.255.255.0:%s" % port, + # "tcp://255.255.255.255:%s" % port, "tcp://\n:%s" % port, ] @@ -263,9 +263,7 @@ def gen_invalid_index_types(): def gen_invalid_nlists(): nlists = [ - 0, -1, - 1000000000000001, # None, [1,2,3], (1,2), @@ -549,4 +547,4 @@ if __name__ == "__main__": p.join() time.sleep(3) status, count = milvus.get_table_row_count(table) - assert count == process_num * loop_num \ No newline at end of file + assert count == process_num * loop_num From 78252ddaca6ac190fa7ca0ccdeec17135452bbe4 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 25 Nov 2019 14:53:42 +0800 Subject: [PATCH 04/18] disable untar verbose --- core/src/index/cmake/BuildUtilsCore.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/index/cmake/BuildUtilsCore.cmake b/core/src/index/cmake/BuildUtilsCore.cmake index 1bf6ba2af4..2261ce3e29 100644 --- a/core/src/index/cmake/BuildUtilsCore.cmake +++ b/core/src/index/cmake/BuildUtilsCore.cmake @@ -74,7 +74,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path) ${CMAKE_COMMAND} -E echo "Extracting ${package_file} to ${install_path}" COMMAND - ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} + ${CMAKE_COMMAND} -E tar xzf ${package_file} ${install_path} WORKING_DIRECTORY ${INDEX_BINARY_DIR} ) From 759a951e4bb4f86b78d46e2c76744fdef05f759f Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Mon, 25 Nov 2019 15:23:50 +0800 Subject: [PATCH 05/18] fix --- core/build.sh | 2 +- .../knowhere/index/vector_index/nsg/NSG.cpp | 60 +++++++++++-------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/core/build.sh b/core/build.sh index 3afb5d1b37..badcc5a032 100755 --- a/core/build.sh +++ b/core/build.sh @@ -56,7 +56,7 @@ while getopts "p:d:t:f:ulrcgjhxzme" arg; do USE_JFROG_CACHE="ON" ;; x) - CUSTOMIZATION="OFF" # force use ori faiss + CUSTOMIZATION="ON" ;; g) GPU_VERSION="ON" diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index bdf538c204..002e160562 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -718,45 +718,55 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co int64_t* ids, SearchParams& params) { std::vector> resset(nq); + params.search_length = k; TimeRecorder rc("search"); - if (nq == 1) { + // TODO(linxj): when to use openmp + if (nq <= 4) { GetNeighbors(query, resset[0], nsg, ¶ms); } else { -//#pragma omp parallel for schedule(dynamic, 50) #pragma omp parallel for for (unsigned int i = 0; i < nq; ++i) { - // TODO(linxj): when to use openmp auto single_query = query + i * dim; GetNeighbors(single_query, resset[i], nsg, ¶ms); } } - rc.ElapseFromBegin("cost"); - + rc.RecordSection("cost"); for (unsigned int i = 0; i < nq; ++i) { - for (unsigned int j = 0; j < k; ++j) { - // ids[i * k + j] = resset[i][j].id; - - // Fix(linxj): bug, reset[i][j] out of range - ids[i * k + j] = ids_[resset[i][j].id]; - dist[i * k + j] = resset[i][j].distance; + int64_t var = resset[i].size() - k; + if (var >= 0) { + for (unsigned int j = 0; j < k; ++j) { + ids[i * k + j] = ids_[resset[i][j].id]; + dist[i * k + j] = resset[i][j].distance; + } + } + else { + for (unsigned int j = 0; j < resset[i].size(); ++j) { + ids[i * k + j] = ids_[resset[i][j].id]; + dist[i * k + j] = resset[i][j].distance; + } + for (unsigned int j = resset[i].size(); j < k; ++j) { + ids[i * k + j] = -1; + dist[i * k + j] = -1; + } } } + rc.RecordSection("merge"); - //>> Debug: test single insert - // int x_0 = resset[0].size(); - // for (int l = 0; l < resset[0].size(); ++l) { - // resset[0].pop_back(); - //} - // resset.clear(); +//>> Debug: test single insert +// int x_0 = resset[0].size(); +// for (int l = 0; l < resset[0].size(); ++l) { +// resset[0].pop_back(); +//} +// resset.clear(); - // ProfilerStart("xx.prof"); - // std::vector resset; - // GetNeighbors(query, resset, nsg, ¶ms); - // for (int i = 0; i < k; ++i) { - // ids[i] = resset[i].id; - // dist[i] = resset[i].distance; - //} - // ProfilerStop(); +// ProfilerStart("xx.prof"); +// std::vector resset; +// GetNeighbors(query, resset, nsg, ¶ms); +// for (int i = 0; i < k; ++i) { +// ids[i] = resset[i].id; +// dist[i] = resset[i].distance; +//} +// ProfilerStop(); } void From 6d0d67ca95a4deec66ffe3add3b2eaa239fd74dd Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Mon, 25 Nov 2019 15:28:25 +0800 Subject: [PATCH 06/18] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53e427dcb9..bc2be5edf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Query API in customization still uses old version - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 +- \#465 - Server hang caused by searching with nsg index - \#486 - gpu no usage during index building ## Feature From 6546bf5af8b97986ea8a542858f4c659a1f3dfb5 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Mon, 25 Nov 2019 15:30:30 +0800 Subject: [PATCH 07/18] format code --- .../knowhere/index/vector_index/nsg/NSG.cpp | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index 002e160562..5872178e96 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -738,8 +738,7 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co ids[i * k + j] = ids_[resset[i][j].id]; dist[i * k + j] = resset[i][j].distance; } - } - else { + } else { for (unsigned int j = 0; j < resset[i].size(); ++j) { ids[i * k + j] = ids_[resset[i][j].id]; dist[i * k + j] = resset[i][j].distance; @@ -752,21 +751,21 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co } rc.RecordSection("merge"); -//>> Debug: test single insert -// int x_0 = resset[0].size(); -// for (int l = 0; l < resset[0].size(); ++l) { -// resset[0].pop_back(); -//} -// resset.clear(); + //>> Debug: test single insert + // int x_0 = resset[0].size(); + // for (int l = 0; l < resset[0].size(); ++l) { + // resset[0].pop_back(); + //} + // resset.clear(); -// ProfilerStart("xx.prof"); -// std::vector resset; -// GetNeighbors(query, resset, nsg, ¶ms); -// for (int i = 0; i < k; ++i) { -// ids[i] = resset[i].id; -// dist[i] = resset[i].distance; -//} -// ProfilerStop(); + // ProfilerStart("xx.prof"); + // std::vector resset; + // GetNeighbors(query, resset, nsg, ¶ms); + // for (int i = 0; i < k; ++i) { + // ids[i] = resset[i].id; + // dist[i] = resset[i].distance; + //} + // ProfilerStop(); } void From f97e62de1e33fc5c55e7529442b9bde89a64a2eb Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 25 Nov 2019 15:31:51 +0800 Subject: [PATCH 08/18] set CMAKE_BUILD_TYPE default values is Release --- core/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 671330c192..b24ac8c00b 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -31,6 +31,10 @@ GET_CURRENT_TIME(BUILD_TIME) string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME}) message(STATUS "Build time = ${BUILD_TIME}") +if (NOT DEFINED CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build.") +endif () + MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'" OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) From d3812c2e5e3838cd8f8034a2e726185bd04315d6 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 25 Nov 2019 16:03:51 +0800 Subject: [PATCH 09/18] update CMakeLists.txt --- core/CMakeLists.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index b24ac8c00b..ba391c2d6b 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -35,9 +35,17 @@ if (NOT DEFINED CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build.") endif () +set (GIT_BRANCH_NAME_REGEX "[0-9]+\\.[0-9]+\\.[0-9]") + MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'" OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}") + execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + endif () + if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}") + execute_process(COMMAND "git" symbolic-ref --short -q HEAD HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + endif () ENDMACRO(GET_GIT_BRANCH_NAME) GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) @@ -47,7 +55,7 @@ if (NOT GIT_BRANCH_NAME STREQUAL "") endif () set(MILVUS_VERSION "${GIT_BRANCH_NAME}") -string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") +string(REGEX MATCH "${GIT_BRANCH_NAME_REGEX}" MILVUS_VERSION "${MILVUS_VERSION}") if (CMAKE_BUILD_TYPE STREQUAL "Release") set(BUILD_TYPE "Release") From 17fda2692261afd9c536803dddf0ad2202f2c18f Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Mon, 25 Nov 2019 16:17:16 +0800 Subject: [PATCH 10/18] fix issue-509 --- CHANGELOG.md | 1 + core/src/wrapper/ConfAdapter.cpp | 30 +++++++++++++++++++----------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53e427dcb9..8012630e0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 - \#486 - gpu no usage during index building +- \#509 - IVF_PQ index build trapped into dead loop caused by invalid params ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index aa4b3c12b8..cf9f1a0474 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -22,6 +22,7 @@ #include #include +#include // TODO(lxj): add conf checker @@ -121,6 +122,13 @@ IVFSQConfAdapter::Match(const TempMetaConf& metaconf) { return conf; } +#define MatchSubQuantizer(c) \ + if (!(conf->d % c)) { \ + WRAPPER_LOG_DEBUG << "PQ m = " << conf->d / c; \ + conf->m = conf->d / c; \ + return conf; \ + } + knowhere::Config IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); @@ -129,18 +137,18 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { conf->metric_type = metaconf.metric_type; conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; - - if (!(conf->d % 4)) - conf->m = conf->d / 4; // compression radio = 16 - else if (!(conf->d % 2)) - conf->m = conf->d / 2; // compression radio = 8 - else if (!(conf->d % 3)) - conf->m = conf->d / 3; // compression radio = 12 - else - conf->m = conf->d; // same as SQ8, compression radio = 4 - MatchBase(conf); - return conf; + + /* + * Faiss 1.6 + * Only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-quantizer are currently supporte with + * no precomputed codes. Precomputed codes supports any number of dimensions, but will involve memory overheads. + */ + static std::vector support_sub_quantizer{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1}; + for (const auto& c : support_sub_quantizer) { + // compression radio = dim / c * 4 + MatchSubQuantizer(c) + } } knowhere::Config From 0c7c4586c059a5fcb4f46bf0003a6bfce596d043 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 25 Nov 2019 16:41:48 +0800 Subject: [PATCH 11/18] #513 DELETE_BY_RANGE sometimes failed --- CHANGELOG.md | 1 + core/src/db/DBImpl.cpp | 14 +++++++++----- core/src/db/DBImpl.h | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53e427dcb9..84bd8aed51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 - \#486 - gpu no usage during index building +- \#513 - DELETE_BY_RANGE sometimes failed ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 51ea665064..1c18c22409 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -105,7 +105,8 @@ DBImpl::Stop() { shutting_down_.store(true, std::memory_order_release); // makesure all memory data serialized - MemSerialize(); + std::set sync_table_ids; + SyncMemData(sync_table_ids); // wait compaction/buildindex finish bg_timer_thread_.join(); @@ -329,7 +330,10 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { return SHUTDOWN_ERROR; } - Status status; + // serialize memory data + std::set sync_table_ids; + auto status = SyncMemData(sync_table_ids); + { std::unique_lock lock(build_index_mutex_); @@ -588,12 +592,12 @@ DBImpl::StartMetricTask() { } Status -DBImpl::MemSerialize() { +DBImpl::SyncMemData(std::set& sync_table_ids) { std::lock_guard lck(mem_serialize_mutex_); std::set temp_table_ids; mem_mgr_->Serialize(temp_table_ids); for (auto& id : temp_table_ids) { - compact_table_ids_.insert(id); + sync_table_ids.insert(id); } if (!temp_table_ids.empty()) { @@ -612,7 +616,7 @@ DBImpl::StartCompactionTask() { } // serialize memory data - MemSerialize(); + SyncMemData(compact_table_ids_); // compactiong has been finished? { diff --git a/core/src/db/DBImpl.h b/core/src/db/DBImpl.h index bff56efded..82a5d3096b 100644 --- a/core/src/db/DBImpl.h +++ b/core/src/db/DBImpl.h @@ -150,7 +150,7 @@ class DBImpl : public DB { BackgroundBuildIndex(); Status - MemSerialize(); + SyncMemData(std::set& sync_table_ids); Status GetFilesToBuildIndex(const std::string& table_id, const std::vector& file_types, From 651b05a06d6471da7fe551acac894fd9cecefbf9 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Mon, 25 Nov 2019 16:44:00 +0800 Subject: [PATCH 12/18] review change --- .../index/knowhere/knowhere/common/Timer.cpp | 37 +++++++------------ .../knowhere/index/vector_index/nsg/NSG.cpp | 4 +- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/core/src/index/knowhere/knowhere/common/Timer.cpp b/core/src/index/knowhere/knowhere/common/Timer.cpp index fefe1e1705..169b55b56f 100644 --- a/core/src/index/knowhere/knowhere/common/Timer.cpp +++ b/core/src/index/knowhere/knowhere/common/Timer.cpp @@ -18,6 +18,7 @@ #include // TODO(linxj): using Log instead #include "knowhere/common/Timer.h" +#include "knowhere/common/Log.h" namespace knowhere { @@ -51,30 +52,18 @@ TimeRecorder::PrintTimeRecord(const std::string& msg, double span) { std::cout << str_log << std::endl; break; } - // case 1: { - // SERVER_LOG_DEBUG << str_log; - // break; - //} - // case 2: { - // SERVER_LOG_INFO << str_log; - // break; - //} - // case 3: { - // SERVER_LOG_WARNING << str_log; - // break; - //} - // case 4: { - // SERVER_LOG_ERROR << str_log; - // break; - //} - // case 5: { - // SERVER_LOG_FATAL << str_log; - // break; - //} - // default: { - // SERVER_LOG_INFO << str_log; - // break; - //} + case 1: { + KNOWHERE_LOG_DEBUG << str_log; + break; + } + // case 2: { + // KNOWHERE_LOG_TRACE << str_log; + // break; + // } + // case 3: { + // KNOWHERE_LOG_WARNING << str_log; + // break; + // } } } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index 5872178e96..b4e00e57b7 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -719,7 +719,7 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co std::vector> resset(nq); params.search_length = k; - TimeRecorder rc("search"); + TimeRecorder rc("NsgIndex::search", 1); // TODO(linxj): when to use openmp if (nq <= 4) { GetNeighbors(query, resset[0], nsg, ¶ms); @@ -730,7 +730,7 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co GetNeighbors(single_query, resset[i], nsg, ¶ms); } } - rc.RecordSection("cost"); + rc.RecordSection("search"); for (unsigned int i = 0; i < nq; ++i) { int64_t var = resset[i].size() - k; if (var >= 0) { From 76aaef7bd79780f7c0d30b6c0a6d04b6669473f9 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Mon, 25 Nov 2019 16:51:11 +0800 Subject: [PATCH 13/18] review change --- core/src/wrapper/ConfAdapter.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index cf9f1a0474..501ada41d6 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -122,13 +122,6 @@ IVFSQConfAdapter::Match(const TempMetaConf& metaconf) { return conf; } -#define MatchSubQuantizer(c) \ - if (!(conf->d % c)) { \ - WRAPPER_LOG_DEBUG << "PQ m = " << conf->d / c; \ - conf->m = conf->d / c; \ - return conf; \ - } - knowhere::Config IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); @@ -146,8 +139,11 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { */ static std::vector support_sub_quantizer{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1}; for (const auto& c : support_sub_quantizer) { - // compression radio = dim / c * 4 - MatchSubQuantizer(c) + if (!(conf->d % c)) { + conf->m = conf->d / c; + WRAPPER_LOG_DEBUG << "PQ m = " << conf->d / c << ", compression radio = " << conf->d / c * 4; + return conf; + } } } From c2971cfd1e97880621570290c0fe005b3761f2d2 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 25 Nov 2019 16:56:41 +0800 Subject: [PATCH 14/18] fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84bd8aed51..060aaed1a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 - \#486 - gpu no usage during index building -- \#513 - DELETE_BY_RANGE sometimes failed +- \#513 - Unittest DELETE_BY_RANGE sometimes failed ## Feature - \#12 - Pure CPU version for Milvus From 60c59d309953ca49e4683d09b6ec07f2d9b07477 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Mon, 25 Nov 2019 18:41:23 +0800 Subject: [PATCH 15/18] format code --- .../index/knowhere/knowhere/common/Timer.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/core/src/index/knowhere/knowhere/common/Timer.cpp b/core/src/index/knowhere/knowhere/common/Timer.cpp index 169b55b56f..141e1ceed6 100644 --- a/core/src/index/knowhere/knowhere/common/Timer.cpp +++ b/core/src/index/knowhere/knowhere/common/Timer.cpp @@ -17,8 +17,8 @@ #include // TODO(linxj): using Log instead -#include "knowhere/common/Timer.h" #include "knowhere/common/Log.h" +#include "knowhere/common/Timer.h" namespace knowhere { @@ -56,14 +56,14 @@ TimeRecorder::PrintTimeRecord(const std::string& msg, double span) { KNOWHERE_LOG_DEBUG << str_log; break; } - // case 2: { - // KNOWHERE_LOG_TRACE << str_log; - // break; - // } - // case 3: { - // KNOWHERE_LOG_WARNING << str_log; - // break; - // } + // case 2: { + // KNOWHERE_LOG_TRACE << str_log; + // break; + // } + // case 3: { + // KNOWHERE_LOG_WARNING << str_log; + // break; + // } } } From cf1e03980edc2eadc9c3b4ad01be34f9c114af51 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Mon, 25 Nov 2019 19:48:47 +0800 Subject: [PATCH 16/18] fix issue 527 --- CHANGELOG.md | 1 + core/src/index/unittest/CMakeLists.txt | 2 +- .../faiss_benchmark/faiss_benchmark_test.cpp | 47 ++++++++++++------- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53e427dcb9..d6db411cf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 - \#486 - gpu no usage during index building +- \#527 - faiss benchmark not compatible with faiss 1.6.0 ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/index/unittest/CMakeLists.txt b/core/src/index/unittest/CMakeLists.txt index 93ae63a9ec..af21d20b48 100644 --- a/core/src/index/unittest/CMakeLists.txt +++ b/core/src/index/unittest/CMakeLists.txt @@ -112,6 +112,6 @@ if (KNOWHERE_GPU_VERSION) install(TARGETS test_customized_index DESTINATION unittest) endif () #add_subdirectory(faiss_ori) -#add_subdirectory(faiss_benchmark) +add_subdirectory(faiss_benchmark) add_subdirectory(test_nsg) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index ce5aa4bafa..0d49004621 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#define USE_FAISS_V0_2_1 0 + #include #include @@ -26,21 +28,6 @@ #include #include -#define USE_FAISS_V1_5_3 0 - -#if USE_FAISS_V1_5_3 -#include -#include -#include -#include -#include - -#else -#include -#include -#include -#endif - #include #include #include @@ -48,8 +35,22 @@ #include #include +#if USE_FAISS_V0_2_1 +#include +#include +#include +#include +#include +#else +#include +#include +#include +#endif + #ifdef CUSTOMIZATION #include +#else +#include #endif /***************************************************** @@ -295,10 +296,12 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std cpu_index = faiss::gpu::index_gpu_to_cpu(gpu_index); delete gpu_index; +#ifdef CUSTOMIZATION faiss::IndexIVF* cpu_ivf_index = dynamic_cast(cpu_index); if (cpu_ivf_index != nullptr) { cpu_ivf_index->to_readonly(); } +#endif printf("[%.3f s] Writing index file: %s\n", elapsed() - t0, index_file_name.c_str()); faiss::write_index(cpu_index, index_file_name.c_str()); @@ -374,13 +377,15 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key faiss::Index *gpu_index, *index; if (query_mode != MODE_CPU) { faiss::gpu::GpuClonerOptions option; +#ifdef CUSTOMIZATION option.allInGpu = true; faiss::IndexComposition index_composition; index_composition.index = cpu_index; index_composition.quantizer = nullptr; - +#endif switch (query_mode) { +#ifdef CUSTOMIZATION case MODE_MIX: { index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data @@ -403,7 +408,9 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key index = cpu_index; break; } +#endif case MODE_GPU: +#ifdef CUSTOMIZATION index_composition.mode = 0; // 0: all data, 1: copy quantizer, 2: copy data // warm up the transmission @@ -412,6 +419,14 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key copy_time = elapsed(); gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option); +#else + // warm up the transmission + gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, cpu_index, &option); + delete gpu_index; + + copy_time = elapsed(); + gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, cpu_index, &option); +#endif copy_time = elapsed() - copy_time; printf("[%.3f s] Copy data completed, cost %f s\n", elapsed() - t0, copy_time); From f25992f596c239a3c1e8b9a9748cb7aa33dc5120 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Mon, 25 Nov 2019 20:01:06 +0800 Subject: [PATCH 17/18] update --- core/src/wrapper/ConfAdapter.cpp | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 501ada41d6..f362808940 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -137,14 +137,30 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { * Only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-quantizer are currently supporte with * no precomputed codes. Precomputed codes supports any number of dimensions, but will involve memory overheads. */ - static std::vector support_sub_quantizer{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1}; - for (const auto& c : support_sub_quantizer) { - if (!(conf->d % c)) { - conf->m = conf->d / c; - WRAPPER_LOG_DEBUG << "PQ m = " << conf->d / c << ", compression radio = " << conf->d / c * 4; - return conf; + static std::vector support_dim_per_subquantizer{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1}; + static std::vector support_subquantizer{96, 64, 56, 48, 40, 32, 28, 24, 20, 16, 12, 8, 4, 3, 2, 1}; + std::vector resset; + for (const auto& dimperquantizer : support_dim_per_subquantizer) { + if (!(conf->d % dimperquantizer)) { + auto subquantzier_num = conf->d / dimperquantizer; + auto finder = std::find(support_subquantizer.begin(), support_subquantizer.end(), subquantzier_num); + if (finder != support_subquantizer.end()) { + resset.push_back(subquantzier_num); + } } } + + if (resset.empty()) { + // todo(linxj): throw exception here. + return nullptr; + } + + static int64_t compression_level = 1; // 1:low, 2:high + if (compression_level == 1) { + conf->m = resset[int(resset.size()/2)]; + WRAPPER_LOG_DEBUG << "PQ m = " << conf->m << ", compression radio = " << conf->d / conf->m * 4; + } + return conf; } knowhere::Config From 1f342f950b55e2ddb48ab2d2e4a96647a63630ee Mon Sep 17 00:00:00 2001 From: Tinkerrr Date: Mon, 25 Nov 2019 22:16:18 +0800 Subject: [PATCH 18/18] format code --- core/src/wrapper/ConfAdapter.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index f362808940..d49747d8f4 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -154,11 +154,10 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { // todo(linxj): throw exception here. return nullptr; } - - static int64_t compression_level = 1; // 1:low, 2:high + static int64_t compression_level = 1; // 1:low, 2:high if (compression_level == 1) { - conf->m = resset[int(resset.size()/2)]; - WRAPPER_LOG_DEBUG << "PQ m = " << conf->m << ", compression radio = " << conf->d / conf->m * 4; + conf->m = resset[int(resset.size() / 2)]; + WRAPPER_LOG_DEBUG << "PQ m = " << conf->m << ", compression radio = " << conf->d / conf->m * 4; } return conf; }