From 5a5e76fe6a5ca3e0b10ec286522f4bd6d1db33a5 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Thu, 28 Nov 2019 14:37:23 +0800 Subject: [PATCH 1/3] #552 Server down during building index_type: IVF_PQ using GPU-edition --- CHANGELOG.md | 1 + core/src/index/unittest/test_nsg/test_nsg.cpp | 2 +- core/src/wrapper/ConfAdapter.cpp | 4 +++- core/unittest/wrapper/test_wrapper.cpp | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 808a019204..d2c33d93ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#533 - NSG build failed with MetricType Inner Product - \#543 - client raise exception in shards when search results is empty - \#545 - Avoid dead circle of build index thread when error occurs +- \#552 - Server down during building index_type: IVF_PQ using GPU-edition ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/index/unittest/test_nsg/test_nsg.cpp b/core/src/index/unittest/test_nsg/test_nsg.cpp index 5de826fabe..c69f4b78fa 100644 --- a/core/src/index/unittest/test_nsg/test_nsg.cpp +++ b/core/src/index/unittest/test_nsg/test_nsg.cpp @@ -42,7 +42,7 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test { SetUp() override { // Init_with_default(); #ifdef MILVUS_GPU_VERSION - knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, 1024 * 1024 * 200, 1024 * 1024 * 600, 2); + knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, 1024 * 1024 * 200, 1024 * 1024 * 600, 1); #endif Generate(256, 1000000 / 100, 1); index_ = std::make_shared(); diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index d49747d8f4..89b9c84b9f 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -152,7 +152,9 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { if (resset.empty()) { // todo(linxj): throw exception here. - return nullptr; + WRAPPER_LOG_ERROR << "The dims of PQ is wrong!"; + throw WrapperException("The dims of PQ is wrong!"); + // return nullptr; } static int64_t compression_level = 1; // 1:low, 2:high if (compression_level == 1) { diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 37e5c31095..2653852494 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -182,6 +182,7 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { TEST(whatever, test_config) { milvus::engine::TempMetaConf conf; conf.nprobe = 16; + conf.dim = 128; auto nsg_conf = std::make_shared(); nsg_conf->Match(conf); nsg_conf->MatchSearch(conf, milvus::engine::IndexType::NSG_MIX); From dd5ec085b6a98efbcd26db85e61ba8020df6ca4b Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Thu, 28 Nov 2019 15:08:05 +0800 Subject: [PATCH 2/3] #552 Server down during building index_type: IVF_PQ using GPU-edition --- core/src/index/unittest/test_nsg/test_nsg.cpp | 3 ++- core/src/wrapper/ConfAdapter.cpp | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/src/index/unittest/test_nsg/test_nsg.cpp b/core/src/index/unittest/test_nsg/test_nsg.cpp index c69f4b78fa..5166b21956 100644 --- a/core/src/index/unittest/test_nsg/test_nsg.cpp +++ b/core/src/index/unittest/test_nsg/test_nsg.cpp @@ -42,7 +42,8 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test { SetUp() override { // Init_with_default(); #ifdef MILVUS_GPU_VERSION - knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, 1024 * 1024 * 200, 1024 * 1024 * 600, 1); + int64_t MB = 1024 * 1024; + knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, MB * 200, MB * 600, 1); #endif Generate(256, 1000000 / 100, 1); index_ = std::make_shared(); diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 89b9c84b9f..03de96a98b 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -134,7 +134,7 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { /* * Faiss 1.6 - * Only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-quantizer are currently supporte with + * Only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-quantizer are currently supported with * no precomputed codes. Precomputed codes supports any number of dimensions, but will involve memory overheads. */ static std::vector support_dim_per_subquantizer{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1}; @@ -152,8 +152,10 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { if (resset.empty()) { // todo(linxj): throw exception here. - WRAPPER_LOG_ERROR << "The dims of PQ is wrong!"; - throw WrapperException("The dims of PQ is wrong!"); + WRAPPER_LOG_ERROR << "The dims of PQ is wrong : only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-" + "quantizer are currently supported with no precomputed codes."; + throw WrapperException("The dims of PQ is wrong : only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims " + "per sub-quantizer are currently supported with no precomputed codes."); // return nullptr; } static int64_t compression_level = 1; // 1:low, 2:high From 035feec8723114d458bba4ae17336e669c238a72 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Thu, 28 Nov 2019 15:15:14 +0800 Subject: [PATCH 3/3] #552 Server down during building index_type: IVF_PQ using GPU-edition --- core/src/wrapper/ConfAdapter.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 03de96a98b..7644e77ef5 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -154,8 +154,9 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { // todo(linxj): throw exception here. WRAPPER_LOG_ERROR << "The dims of PQ is wrong : only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-" "quantizer are currently supported with no precomputed codes."; - throw WrapperException("The dims of PQ is wrong : only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims " - "per sub-quantizer are currently supported with no precomputed codes."); + throw WrapperException( + "The dims of PQ is wrong : only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims " + "per sub-quantizer are currently supported with no precomputed codes."); // return nullptr; } static int64_t compression_level = 1; // 1:low, 2:high