From bee37170f4b755f7b739152a6c24ddfdf4e202e5 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Mon, 2 Dec 2019 14:40:16 +0800 Subject: [PATCH 1/6] IP search metric_type shouldn't be supported by IVF_PQ index on GPU version #616 #636 --- CHANGELOG.md | 2 ++ .../knowhere/index/vector_index/IndexGPUIVFPQ.cpp | 4 ++++ core/src/wrapper/gpu/GPUVecImpl.cpp | 2 ++ core/unittest/db/test_db.cpp | 13 +++++++++++++ 4 files changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19a9c4519d..7956e513fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,9 @@ Please mark all change in change log and use the ticket from JIRA. - \#599 - Build index log is incorrect - \#602 - Optimizer specify wrong gpu_id - \#606 - No log generated during building index with CPU +- \#616 - IP search metric_type is not supported by IVF_PQ index - \#631 - FAISS isn't compiled with O3 option +- \#636 - [CPU] Create index PQ should be failed if table metric type set Inner Product ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp index cbd4f4f09c..a4693d8035 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -32,6 +32,10 @@ namespace knowhere { IndexModelPtr GPUIVFPQ::Train(const DatasetPtr& dataset, const Config& config) { auto build_cfg = std::dynamic_pointer_cast(config); + if (build_cfg->metric_type == knowhere::METRICTYPE::IP) { + KNOWHERE_LOG_ERROR << "PQ not support IP in GPU version!"; + throw KnowhereException("PQ not support IP in GPU version!"); + } if (build_cfg != nullptr) { build_cfg->CheckValid(); // throw exception } diff --git a/core/src/wrapper/gpu/GPUVecImpl.cpp b/core/src/wrapper/gpu/GPUVecImpl.cpp index e40f5038ea..99fa7589d1 100644 --- a/core/src/wrapper/gpu/GPUVecImpl.cpp +++ b/core/src/wrapper/gpu/GPUVecImpl.cpp @@ -16,6 +16,7 @@ // under the License. #include "wrapper/gpu/GPUVecImpl.h" +#include #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexGPUIDMAP.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" @@ -58,6 +59,7 @@ IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, co } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); + throw WrapperException(e.what()); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (std::exception& e) { WRAPPER_LOG_ERROR << e.what(); diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index 0a47ac1b9b..6c14b86b39 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -336,6 +336,19 @@ TEST_F(DBTest, SEARCH_TEST) { } #endif + index.engine_type_ = (int)milvus::engine::EngineType::FAISS_PQ; + db_->CreateIndex(TABLE_NAME, index); // wait until build index finish + + { + std::vector tags; + milvus::engine::ResultIds result_ids; + milvus::engine::ResultDistances result_distances; + stat = db_->Query(TABLE_NAME, tags, k, nq, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + stat = db_->Query(TABLE_NAME, tags, k, 1100, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + } + { // search by specify index file milvus::engine::meta::DatesT dates; std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; From 36d84f374bfc742a014d5ed808d0d52698a7f73a Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Mon, 2 Dec 2019 17:02:03 +0800 Subject: [PATCH 2/6] change for comment --- core/src/wrapper/gpu/GPUVecImpl.cpp | 3 +-- core/unittest/db/test_db.cpp | 18 +++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/core/src/wrapper/gpu/GPUVecImpl.cpp b/core/src/wrapper/gpu/GPUVecImpl.cpp index 99fa7589d1..167a4d6a98 100644 --- a/core/src/wrapper/gpu/GPUVecImpl.cpp +++ b/core/src/wrapper/gpu/GPUVecImpl.cpp @@ -16,7 +16,6 @@ // under the License. #include "wrapper/gpu/GPUVecImpl.h" -#include #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexGPUIDMAP.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" @@ -26,6 +25,7 @@ #include "src/wrapper/DataTransfer.h" #include "utils/Log.h" #include "wrapper/VecImpl.h" +#include "wrapper/WrapperException.h" /* * no parameter check in this layer. @@ -60,7 +60,6 @@ IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, co } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); throw WrapperException(e.what()); - return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (std::exception& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index 6c14b86b39..8d05ae9923 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -295,6 +295,15 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_TRUE(stat.ok()); } + { // search by specify index file + milvus::engine::meta::DatesT dates; + std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; + milvus::engine::ResultIds result_ids; + milvus::engine::ResultDistances result_distances; + stat = db_->QueryByFileID(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + } + index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFFLAT; db_->CreateIndex(TABLE_NAME, index); // wait until build index finish @@ -349,15 +358,6 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_TRUE(stat.ok()); } - { // search by specify index file - milvus::engine::meta::DatesT dates; - std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; - milvus::engine::ResultIds result_ids; - milvus::engine::ResultDistances result_distances; - stat = db_->QueryByFileID(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, result_ids, result_distances); - ASSERT_TRUE(stat.ok()); - } - #ifdef CUSTOMIZATION // test FAISS_IVFSQ8H optimizer index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H; From b1b086a03f1eda8947cfeb43960289d548f0273a Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Mon, 2 Dec 2019 17:47:17 +0800 Subject: [PATCH 3/6] remove change of db_test --- core/unittest/db/test_db.cpp | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index 8d05ae9923..0a47ac1b9b 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -295,15 +295,6 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_TRUE(stat.ok()); } - { // search by specify index file - milvus::engine::meta::DatesT dates; - std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; - milvus::engine::ResultIds result_ids; - milvus::engine::ResultDistances result_distances; - stat = db_->QueryByFileID(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, result_ids, result_distances); - ASSERT_TRUE(stat.ok()); - } - index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFFLAT; db_->CreateIndex(TABLE_NAME, index); // wait until build index finish @@ -345,16 +336,12 @@ TEST_F(DBTest, SEARCH_TEST) { } #endif - index.engine_type_ = (int)milvus::engine::EngineType::FAISS_PQ; - db_->CreateIndex(TABLE_NAME, index); // wait until build index finish - - { - std::vector tags; + { // search by specify index file + milvus::engine::meta::DatesT dates; + std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(TABLE_NAME, tags, k, nq, 10, xq.data(), result_ids, result_distances); - ASSERT_TRUE(stat.ok()); - stat = db_->Query(TABLE_NAME, tags, k, 1100, 10, xq.data(), result_ids, result_distances); + stat = db_->QueryByFileID(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, result_ids, result_distances); ASSERT_TRUE(stat.ok()); } From fcc98ca183eeaf053101c8562a32c36f98bbd373 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Mon, 2 Dec 2019 21:47:11 +0800 Subject: [PATCH 4/6] IP search metric_type shouldn't be supported by IVF_PQ index on GPU version --- .../knowhere/index/vector_index/IndexGPUIVFPQ.cpp | 4 ---- core/src/wrapper/ConfAdapter.cpp | 12 ++++++++++++ core/src/wrapper/gpu/GPUVecImpl.cpp | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp index a4693d8035..cbd4f4f09c 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -32,10 +32,6 @@ namespace knowhere { IndexModelPtr GPUIVFPQ::Train(const DatasetPtr& dataset, const Config& config) { auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg->metric_type == knowhere::METRICTYPE::IP) { - KNOWHERE_LOG_ERROR << "PQ not support IP in GPU version!"; - throw KnowhereException("PQ not support IP in GPU version!"); - } if (build_cfg != nullptr) { build_cfg->CheckValid(); // throw exception } diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 6b1667f9d7..0214025ed7 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -18,6 +18,7 @@ #include "wrapper/ConfAdapter.h" #include "WrapperException.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" +#include "server/Config.h" #include "utils/Log.h" #include @@ -130,6 +131,17 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { conf->nbits = 8; MatchBase(conf); +#ifdef MILVUS_GPU_VERSION + Status s; + bool enable_gpu = false; + server::Config& config = server::Config::GetInstance(); + s = config.GetGpuResourceConfigEnable(enable_gpu); + if (s.ok() && conf->metric_type == knowhere::METRICTYPE::IP) { + WRAPPER_LOG_ERROR << "PQ not support IP in GPU version!"; + throw WrapperException("PQ not support IP in GPU version!"); + } +#endif + /* * Faiss 1.6 * Only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-quantizer are currently supported with diff --git a/core/src/wrapper/gpu/GPUVecImpl.cpp b/core/src/wrapper/gpu/GPUVecImpl.cpp index 167a4d6a98..500bd61b9b 100644 --- a/core/src/wrapper/gpu/GPUVecImpl.cpp +++ b/core/src/wrapper/gpu/GPUVecImpl.cpp @@ -59,7 +59,7 @@ IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, co } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); - throw WrapperException(e.what()); + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (std::exception& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); From d69e30ebfc3495b4053cd304ad202100a878bd81 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Mon, 2 Dec 2019 21:48:49 +0800 Subject: [PATCH 5/6] remove wrapperexception --- core/src/wrapper/gpu/GPUVecImpl.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/wrapper/gpu/GPUVecImpl.cpp b/core/src/wrapper/gpu/GPUVecImpl.cpp index 500bd61b9b..e40f5038ea 100644 --- a/core/src/wrapper/gpu/GPUVecImpl.cpp +++ b/core/src/wrapper/gpu/GPUVecImpl.cpp @@ -25,7 +25,6 @@ #include "src/wrapper/DataTransfer.h" #include "utils/Log.h" #include "wrapper/VecImpl.h" -#include "wrapper/WrapperException.h" /* * no parameter check in this layer. From 615d1ac2a64de855dcb55a842dde3fe76f5785c9 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Tue, 3 Dec 2019 12:54:36 +0800 Subject: [PATCH 6/6] fix build pq with empty table --- .../grpc_impl/request/CreateIndexRequest.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/core/src/server/grpc_impl/request/CreateIndexRequest.cpp b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp index c7628048ff..0c18bbe42f 100644 --- a/core/src/server/grpc_impl/request/CreateIndexRequest.cpp +++ b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp @@ -16,6 +16,7 @@ // under the License. #include "server/grpc_impl/request/CreateIndexRequest.h" +#include "server/Config.h" #include "server/DBWrapper.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" @@ -75,6 +76,20 @@ CreateIndexRequest::OnExecute() { return status; } +#ifdef MILVUS_GPU_VERSION + Status s; + bool enable_gpu = false; + server::Config& config = server::Config::GetInstance(); + s = config.GetGpuResourceConfigEnable(enable_gpu); + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name_; + status = DBWrapper::DB()->DescribeTable(table_info); + if (s.ok() && grpc_index.index_type() == (int)engine::EngineType::FAISS_PQ && + table_info.metric_type_ == (int)engine::MetricType::IP) { + return Status(SERVER_UNEXPECTED_ERROR, "PQ not support IP in GPU version!"); + } +#endif + // step 2: check table existence engine::TableIndex index; index.engine_type_ = grpc_index.index_type();