From f4ebd3a9ce8be3fd3e13ffa7dd2951d6a1ccad3b Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Fri, 25 Mar 2022 13:49:25 +0800 Subject: [PATCH] Upgrade to knowhere v1.1.0 (#16186) Signed-off-by: yudong.cai --- internal/core/bench/bench_indexbuilder.cpp | 12 +- internal/core/src/common/LoadInfo.h | 2 +- internal/core/src/common/vector_index_c.cpp | 14 +- internal/core/src/config/ConfigKnowhere.cpp | 27 +- internal/core/src/index/Index.h | 2 +- .../src/indexbuilder/ScalarIndexCreator-inl.h | 4 +- .../src/indexbuilder/ScalarIndexCreator.h | 2 +- .../core/src/indexbuilder/VecIndexCreator.cpp | 74 ++-- .../core/src/indexbuilder/VecIndexCreator.h | 6 +- internal/core/src/indexbuilder/index_c.cpp | 18 +- internal/core/src/indexbuilder/utils.cpp | 26 +- internal/core/src/indexbuilder/utils.h | 10 +- internal/core/src/query/SearchBruteForce.cpp | 2 +- internal/core/src/query/SearchOnIndex.cpp | 4 +- internal/core/src/query/SearchOnSealed.cpp | 6 +- internal/core/src/segcore/load_index_c.cpp | 11 +- .../core/thirdparty/knowhere/CMakeLists.txt | 6 +- internal/core/unittest/test_c_api.cpp | 332 +++++++++--------- .../core/unittest/test_conf_adapter_mgr.cpp | 4 +- internal/core/unittest/test_index_c_api.cpp | 22 +- internal/core/unittest/test_index_wrapper.cpp | 213 +++++------ internal/core/unittest/test_indexing.cpp | 20 +- .../unittest/test_scalar_index_creator.cpp | 20 +- internal/core/unittest/test_sealed.cpp | 10 +- internal/core/unittest/test_utils/DataGen.h | 2 +- .../test_utils/indexbuilder_test_utils.h | 306 ++++++++-------- internal/util/indexcgowrapper/index_test.go | 12 +- 27 files changed, 587 insertions(+), 580 deletions(-) diff --git a/internal/core/bench/bench_indexbuilder.cpp b/internal/core/bench/bench_indexbuilder.cpp index 570294fc43..f5a25bd030 100644 --- a/internal/core/bench/bench_indexbuilder.cpp +++ b/internal/core/bench/bench_indexbuilder.cpp @@ -27,15 +27,15 @@ constexpr int64_t NB = 1000000; namespace indexcgo = milvus::proto::indexcgo; auto index_type_collections = [] { - static std::map collections{ - {0, milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT}, + static std::map collections{ + {0, knowhere::IndexEnum::INDEX_FAISS_IVFFLAT}, }; return collections; }(); auto metric_type_collections = [] { - static std::map collections{ - {0, milvus::knowhere::Metric::L2}, + static std::map collections{ + {0, knowhere::Metric::L2}, }; return collections; }(); @@ -60,7 +60,7 @@ IndexBuilder_build(benchmark::State& state) { auto is_binary = state.range(2); auto dataset = GenDataset(NB, metric_type, is_binary); auto xb_data = dataset.get_col(0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); for (auto _ : state) { auto index = @@ -89,7 +89,7 @@ IndexBuilder_build_and_codec(benchmark::State& state) { auto is_binary = state.range(2); auto dataset = GenDataset(NB, metric_type, is_binary); auto xb_data = dataset.get_col(0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); for (auto _ : state) { auto index = diff --git a/internal/core/src/common/LoadInfo.h b/internal/core/src/common/LoadInfo.h index b4e5c1a6db..dd5ad4dd78 100644 --- a/internal/core/src/common/LoadInfo.h +++ b/internal/core/src/common/LoadInfo.h @@ -24,7 +24,7 @@ struct LoadIndexInfo { int64_t field_id; std::map index_params; - milvus::knowhere::VecIndexPtr index; + knowhere::VecIndexPtr index; }; // NOTE: field_id can be system field diff --git a/internal/core/src/common/vector_index_c.cpp b/internal/core/src/common/vector_index_c.cpp index 14a8745bf0..d632706c0f 100644 --- a/internal/core/src/common/vector_index_c.cpp +++ b/internal/core/src/common/vector_index_c.cpp @@ -20,7 +20,7 @@ CStatus NewBinarySet(CBinarySet* c_binary_set) { try { - auto binary_set = std::make_unique(); + auto binary_set = std::make_unique(); *c_binary_set = binary_set.release(); auto status = CStatus(); status.error_code = Success; @@ -36,7 +36,7 @@ NewBinarySet(CBinarySet* c_binary_set) { void DeleteBinarySet(CBinarySet c_binary_set) { - auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + auto binary_set = (knowhere::BinarySet*)c_binary_set; delete binary_set; } @@ -44,7 +44,7 @@ CStatus AppendIndexBinary(CBinarySet c_binary_set, void* index_binary, int64_t index_size, const char* c_index_key) { auto status = CStatus(); try { - auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + auto binary_set = (knowhere::BinarySet*)c_binary_set; std::string index_key(c_index_key); uint8_t* index = (uint8_t*)index_binary; uint8_t* dup = new uint8_t[index_size](); @@ -63,13 +63,13 @@ AppendIndexBinary(CBinarySet c_binary_set, void* index_binary, int64_t index_siz int GetBinarySetSize(CBinarySet c_binary_set) { - auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + auto binary_set = (knowhere::BinarySet*)c_binary_set; return binary_set->binary_map_.size(); } void GetBinarySetKeys(CBinarySet c_binary_set, void* datas) { - auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + auto binary_set = (knowhere::BinarySet*)c_binary_set; auto& map_ = binary_set->binary_map_; const char** datas_ = (const char**)datas; std::size_t i = 0; @@ -80,7 +80,7 @@ GetBinarySetKeys(CBinarySet c_binary_set, void* datas) { int GetBinarySetValueSize(CBinarySet c_binary_set, const char* key) { - auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + auto binary_set = (knowhere::BinarySet*)c_binary_set; int64_t ret_ = 0; try { std::string key_(key); @@ -94,7 +94,7 @@ GetBinarySetValueSize(CBinarySet c_binary_set, const char* key) { CStatus CopyBinarySetValue(void* data, const char* key, CBinarySet c_binary_set) { auto status = CStatus(); - auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + auto binary_set = (knowhere::BinarySet*)c_binary_set; try { auto binary = binary_set->GetByName(key); status.error_code = Success; diff --git a/internal/core/src/config/ConfigKnowhere.cpp b/internal/core/src/config/ConfigKnowhere.cpp index e08c9b6319..8fc42c3eff 100644 --- a/internal/core/src/config/ConfigKnowhere.cpp +++ b/internal/core/src/config/ConfigKnowhere.cpp @@ -22,19 +22,17 @@ #include "log/Log.h" #include "knowhere/archive/KnowhereConfig.h" -namespace milvus { -namespace config { +namespace milvus::config { std::once_flag init_knowhere_once_; void KnowhereInitImpl() { auto init = []() { - namespace eg = milvus::engine; - eg::KnowhereConfig::SetBlasThreshold(16384); - eg::KnowhereConfig::SetEarlyStopThreshold(0); - eg::KnowhereConfig::SetLogHandler(); - eg::KnowhereConfig::SetStatisticsLevel(0); + knowhere::KnowhereConfig::SetBlasThreshold(16384); + knowhere::KnowhereConfig::SetEarlyStopThreshold(0); + knowhere::KnowhereConfig::SetLogHandler(); + knowhere::KnowhereConfig::SetStatisticsLevel(0); el::Configurations el_conf; el_conf.setGlobally(el::ConfigurationType::Enabled, std::to_string(false)); }; @@ -44,25 +42,24 @@ KnowhereInitImpl() { std::string KnowhereSetSimdType(const char* value) { - milvus::engine::KnowhereConfig::SimdType simd_type; + knowhere::KnowhereConfig::SimdType simd_type; if (strcmp(value, "auto") == 0) { - simd_type = milvus::engine::KnowhereConfig::SimdType::AUTO; + simd_type = knowhere::KnowhereConfig::SimdType::AUTO; } else if (strcmp(value, "avx512") == 0) { - simd_type = milvus::engine::KnowhereConfig::SimdType::AVX512; + simd_type = knowhere::KnowhereConfig::SimdType::AVX512; } else if (strcmp(value, "avx2") == 0) { - simd_type = milvus::engine::KnowhereConfig::SimdType::AVX2; + simd_type = knowhere::KnowhereConfig::SimdType::AVX2; } else if (strcmp(value, "avx") == 0 || strcmp(value, "sse4_2") == 0) { - simd_type = milvus::engine::KnowhereConfig::SimdType::SSE4_2; + simd_type = knowhere::KnowhereConfig::SimdType::SSE4_2; } else { PanicInfo("invalid SIMD type: " + std::string(value)); } try { - return milvus::engine::KnowhereConfig::SetSimdType(simd_type); + return knowhere::KnowhereConfig::SetSimdType(simd_type); } catch (std::exception& e) { LOG_SERVER_ERROR_ << e.what(); PanicInfo(e.what()); } } -} // namespace config -} // namespace milvus +} // namespace milvus::config diff --git a/internal/core/src/index/Index.h b/internal/core/src/index/Index.h index 39b57ccbb8..2dcf089ae1 100644 --- a/internal/core/src/index/Index.h +++ b/internal/core/src/index/Index.h @@ -17,7 +17,7 @@ #include namespace milvus::scalar { -using Index = milvus::knowhere::Index; +using Index = knowhere::Index; using IndexPtr = std::unique_ptr; using BinarySet = knowhere::BinarySet; using Config = knowhere::Config; diff --git a/internal/core/src/indexbuilder/ScalarIndexCreator-inl.h b/internal/core/src/indexbuilder/ScalarIndexCreator-inl.h index bb98604656..1268f38c3c 100644 --- a/internal/core/src/indexbuilder/ScalarIndexCreator-inl.h +++ b/internal/core/src/indexbuilder/ScalarIndexCreator-inl.h @@ -52,7 +52,7 @@ ScalarIndexCreator::Load(const knowhere::BinarySet& binary_set) { // not sure that the pointer of a golang bool array acts like other types. template <> inline void -ScalarIndexCreator::Build(const milvus::knowhere::DatasetPtr& dataset) { +ScalarIndexCreator::Build(const knowhere::DatasetPtr& dataset) { auto size = dataset->Get(knowhere::meta::ROWS); auto data = dataset->Get(knowhere::meta::TENSOR); proto::schema::BoolArray arr; @@ -71,7 +71,7 @@ inline ScalarIndexCreator::ScalarIndexCreator(const char* type_para template <> inline void -ScalarIndexCreator::Build(const milvus::knowhere::DatasetPtr& dataset) { +ScalarIndexCreator::Build(const knowhere::DatasetPtr& dataset) { auto size = dataset->Get(knowhere::meta::ROWS); auto data = dataset->Get(knowhere::meta::TENSOR); proto::schema::StringArray arr; diff --git a/internal/core/src/indexbuilder/ScalarIndexCreator.h b/internal/core/src/indexbuilder/ScalarIndexCreator.h index 7a61d5b5e8..052fd999bc 100644 --- a/internal/core/src/indexbuilder/ScalarIndexCreator.h +++ b/internal/core/src/indexbuilder/ScalarIndexCreator.h @@ -42,7 +42,7 @@ class ScalarIndexCreator : public IndexCreatorBase { std::unique_ptr> index_ = nullptr; proto::indexcgo::TypeParams type_params_; proto::indexcgo::IndexParams index_params_; - milvus::json config_; + knowhere::Config config_; }; } // namespace milvus::indexbuilder diff --git a/internal/core/src/indexbuilder/VecIndexCreator.cpp b/internal/core/src/indexbuilder/VecIndexCreator.cpp index 1e4a6d89ff..e6d616ee66 100644 --- a/internal/core/src/indexbuilder/VecIndexCreator.cpp +++ b/internal/core/src/indexbuilder/VecIndexCreator.cpp @@ -65,50 +65,50 @@ VecIndexCreator::parse_impl(const std::string& serialized_params_str, knowhere:: }; /***************************** meta *******************************/ - check_parameter(conf, milvus::knowhere::meta::DIM, stoi_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::meta::TOPK, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::meta::DIM, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::meta::TOPK, stoi_closure, std::nullopt); /***************************** IVF Params *******************************/ - check_parameter(conf, milvus::knowhere::IndexParams::nprobe, stoi_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::IndexParams::nlist, stoi_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::IndexParams::m, stoi_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::IndexParams::nbits, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::nprobe, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::nlist, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::m, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::nbits, stoi_closure, std::nullopt); /************************** NSG Parameter **************************/ - check_parameter(conf, milvus::knowhere::IndexParams::knng, stoi_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::IndexParams::search_length, stoi_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::IndexParams::out_degree, stoi_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::IndexParams::candidate, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::knng, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::search_length, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::out_degree, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::candidate, stoi_closure, std::nullopt); /************************** HNSW Params *****************************/ - check_parameter(conf, milvus::knowhere::IndexParams::efConstruction, stoi_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::IndexParams::M, stoi_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::IndexParams::ef, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::efConstruction, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::M, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::ef, stoi_closure, std::nullopt); /************************** Annoy Params *****************************/ - check_parameter(conf, milvus::knowhere::IndexParams::n_trees, stoi_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::IndexParams::search_k, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::n_trees, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::search_k, stoi_closure, std::nullopt); /************************** PQ Params *****************************/ - check_parameter(conf, milvus::knowhere::IndexParams::PQM, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::PQM, stoi_closure, std::nullopt); /************************** NGT Params *****************************/ - check_parameter(conf, milvus::knowhere::IndexParams::edge_size, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::edge_size, stoi_closure, std::nullopt); /************************** NGT Search Params *****************************/ - check_parameter(conf, milvus::knowhere::IndexParams::epsilon, stof_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::IndexParams::max_search_edges, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::epsilon, stof_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::max_search_edges, stoi_closure, std::nullopt); /************************** NGT_PANNG Params *****************************/ - check_parameter(conf, milvus::knowhere::IndexParams::forcedly_pruned_edge_size, stoi_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::IndexParams::selectively_pruned_edge_size, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::forcedly_pruned_edge_size, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::selectively_pruned_edge_size, stoi_closure, std::nullopt); /************************** NGT_ONNG Params *****************************/ - check_parameter(conf, milvus::knowhere::IndexParams::outgoing_edge_size, stoi_closure, std::nullopt); - check_parameter(conf, milvus::knowhere::IndexParams::incoming_edge_size, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::outgoing_edge_size, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::IndexParams::incoming_edge_size, stoi_closure, std::nullopt); /************************** Serialize Params *******************************/ - check_parameter(conf, milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, stoi_closure, std::optional{4}); + check_parameter(conf, knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, stoi_closure, std::optional{4}); } void @@ -149,7 +149,7 @@ VecIndexCreator::get_config_by_name(std::string name) { int64_t VecIndexCreator::dim() { - auto dimension = get_config_by_name(milvus::knowhere::meta::DIM); + auto dimension = get_config_by_name(knowhere::meta::DIM); AssertInfo(dimension.has_value(), "[VecIndexCreator]Dimension doesn't have value"); return (dimension.value()); } @@ -190,7 +190,7 @@ VecIndexCreator::BuildWithoutIds(const knowhere::DatasetPtr& dataset) { void VecIndexCreator::BuildWithIds(const knowhere::DatasetPtr& dataset) { - AssertInfo(dataset->data().find(milvus::knowhere::meta::IDS) != dataset->data().end(), + AssertInfo(dataset->data().find(knowhere::meta::IDS) != dataset->data().end(), "[VecIndexCreator]Can't find ids field in dataset"); auto index_type = get_index_type(); auto index_mode = get_index_mode(); @@ -215,9 +215,9 @@ void VecIndexCreator::StoreRawData(const knowhere::DatasetPtr& dataset) { auto index_type = get_index_type(); if (is_in_nm_list(index_type)) { - auto tensor = dataset->Get(milvus::knowhere::meta::TENSOR); - auto row_num = dataset->Get(milvus::knowhere::meta::ROWS); - auto dim = dataset->Get(milvus::knowhere::meta::DIM); + auto tensor = dataset->Get(knowhere::meta::TENSOR); + auto row_num = dataset->Get(knowhere::meta::ROWS); + auto dim = dataset->Get(knowhere::meta::DIM); int64_t data_size; if (is_in_bin_list(index_type)) { data_size = dim / 8 * row_num; @@ -229,7 +229,7 @@ VecIndexCreator::StoreRawData(const knowhere::DatasetPtr& dataset) { } } -milvus::knowhere::BinarySet +knowhere::BinarySet VecIndexCreator::Serialize() { auto ret = index_->Serialize(config_); auto index_type = get_index_type(); @@ -247,7 +247,7 @@ VecIndexCreator::Serialize() { } void -VecIndexCreator::Load(const milvus::knowhere::BinarySet& binary_set) { +VecIndexCreator::Load(const knowhere::BinarySet& binary_set) { auto& map_ = binary_set.binary_map_; for (auto it = map_.begin(); it != map_.end(); ++it) { if (it->first == RAW_DATA) { @@ -311,7 +311,7 @@ VecIndexCreator::Query(const knowhere::DatasetPtr& dataset) { std::unique_ptr VecIndexCreator::QueryWithParam(const knowhere::DatasetPtr& dataset, const char* serialized_search_params) { namespace indexcgo = milvus::proto::indexcgo; - milvus::knowhere::Config search_conf; + knowhere::Config search_conf; parse_impl(std::string(serialized_search_params), search_conf); return std::move(QueryImpl(dataset, search_conf)); @@ -326,10 +326,10 @@ VecIndexCreator::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere:: } auto res = index_->Query(dataset, conf, nullptr); - auto ids = res->Get(milvus::knowhere::meta::IDS); - auto distances = res->Get(milvus::knowhere::meta::DISTANCE); - auto nq = dataset->Get(milvus::knowhere::meta::ROWS); - auto k = config_[milvus::knowhere::meta::TOPK].get(); + auto ids = res->Get(knowhere::meta::IDS); + auto distances = res->Get(knowhere::meta::DISTANCE); + auto nq = dataset->Get(knowhere::meta::ROWS); + auto k = config_[knowhere::meta::TOPK].get(); auto query_res = std::make_unique(); query_res->nq = nq; @@ -347,7 +347,7 @@ VecIndexCreator::LoadRawData() { auto index_type = get_index_type(); if (is_in_nm_list(index_type)) { auto bs = index_->Serialize(config_); - auto bptr = std::make_shared(); + auto bptr = std::make_shared(); auto deleter = [&](uint8_t*) {}; // avoid repeated deconstruction bptr->data = std::shared_ptr(static_cast(raw_data_.data()), deleter); bptr->size = raw_data_.size(); diff --git a/internal/core/src/indexbuilder/VecIndexCreator.h b/internal/core/src/indexbuilder/VecIndexCreator.h index 63b7d37da9..0e1e9b5eb8 100644 --- a/internal/core/src/indexbuilder/VecIndexCreator.h +++ b/internal/core/src/indexbuilder/VecIndexCreator.h @@ -44,7 +44,7 @@ class VecIndexCreator : public IndexCreatorBase { public: // used for tests struct QueryResult { - std::vector ids; + std::vector ids; std::vector distances; int64_t nq; int64_t topk; @@ -107,8 +107,8 @@ class VecIndexCreator : public IndexCreatorBase { knowhere::VecIndexPtr index_ = nullptr; std::string type_params_; std::string index_params_; - milvus::json type_config_; - milvus::json index_config_; + knowhere::Config type_config_; + knowhere::Config index_config_; knowhere::Config config_; std::vector raw_data_; std::once_flag raw_data_loaded_; diff --git a/internal/core/src/indexbuilder/index_c.cpp b/internal/core/src/indexbuilder/index_c.cpp index b38bcc7486..6800009cd3 100644 --- a/internal/core/src/indexbuilder/index_c.cpp +++ b/internal/core/src/indexbuilder/index_c.cpp @@ -73,7 +73,7 @@ BuildFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors) auto cIndex = dynamic_cast(real_index); auto dim = cIndex->dim(); auto row_nums = float_value_num / dim; - auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); + auto ds = knowhere::GenDataset(row_nums, dim, vectors); cIndex->Build(ds); status.error_code = Success; status.error_msg = ""; @@ -93,7 +93,7 @@ BuildBinaryVecIndex(CIndex index, int64_t data_size, const uint8_t* vectors) { auto cIndex = dynamic_cast(real_index); auto dim = cIndex->dim(); auto row_nums = (data_size * 8) / dim; - auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); + auto ds = knowhere::GenDataset(row_nums, dim, vectors); cIndex->Build(ds); status.error_code = Success; status.error_msg = ""; @@ -117,7 +117,7 @@ BuildScalarIndex(CIndex c_index, int64_t size, const void* field_data) { auto real_index = reinterpret_cast(c_index); const int64_t dim = 8; // not important here - auto dataset = milvus::knowhere::GenDataset(size, dim, field_data); + auto dataset = knowhere::GenDataset(size, dim, field_data); real_index->Build(dataset); status.error_code = Success; @@ -135,7 +135,7 @@ SerializeIndexToBinarySet(CIndex index, CBinarySet* c_binary_set) { try { AssertInfo(index, "failed to serialize index to binary set, passed index was null"); auto real_index = reinterpret_cast(index); - auto binary = std::make_unique(real_index->Serialize()); + auto binary = std::make_unique(real_index->Serialize()); *c_binary_set = binary.release(); status.error_code = Success; status.error_msg = ""; @@ -152,7 +152,7 @@ LoadIndexFromBinarySet(CIndex index, CBinarySet c_binary_set) { try { AssertInfo(index, "failed to load index from binary set, passed index was null"); auto real_index = reinterpret_cast(index); - auto binary_set = reinterpret_cast(c_binary_set); + auto binary_set = reinterpret_cast(c_binary_set); real_index->Load(*binary_set); status.error_code = Success; status.error_msg = ""; @@ -170,7 +170,7 @@ QueryOnFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors auto cIndex = (milvus::indexbuilder::VecIndexCreator*)index; auto dim = cIndex->dim(); auto row_nums = float_value_num / dim; - auto query_ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); + auto query_ds = knowhere::GenDataset(row_nums, dim, vectors); auto query_res = cIndex->Query(query_ds); *res = query_res.release(); @@ -194,7 +194,7 @@ QueryOnFloatVecIndexWithParam(CIndex index, auto cIndex = (milvus::indexbuilder::VecIndexCreator*)index; auto dim = cIndex->dim(); auto row_nums = float_value_num / dim; - auto query_ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); + auto query_ds = knowhere::GenDataset(row_nums, dim, vectors); auto query_res = cIndex->QueryWithParam(query_ds, serialized_search_params); *res = query_res.release(); @@ -214,7 +214,7 @@ QueryOnBinaryVecIndex(CIndex index, int64_t data_size, const uint8_t* vectors, C auto cIndex = (milvus::indexbuilder::VecIndexCreator*)index; auto dim = cIndex->dim(); auto row_nums = (data_size * 8) / dim; - auto query_ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); + auto query_ds = knowhere::GenDataset(row_nums, dim, vectors); auto query_res = cIndex->Query(query_ds); *res = query_res.release(); @@ -238,7 +238,7 @@ QueryOnBinaryVecIndexWithParam(CIndex index, auto cIndex = (milvus::indexbuilder::VecIndexCreator*)index; auto dim = cIndex->dim(); auto row_nums = (data_size * 8) / dim; - auto query_ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); + auto query_ds = knowhere::GenDataset(row_nums, dim, vectors); auto query_res = cIndex->QueryWithParam(query_ds, serialized_search_params); *res = query_res.release(); diff --git a/internal/core/src/indexbuilder/utils.cpp b/internal/core/src/indexbuilder/utils.cpp index 2b576d2904..28648609d2 100644 --- a/internal/core/src/indexbuilder/utils.cpp +++ b/internal/core/src/indexbuilder/utils.cpp @@ -24,9 +24,9 @@ namespace milvus::indexbuilder { std::vector NM_List() { static std::vector ret{ - milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, - milvus::knowhere::IndexEnum::INDEX_NSG, - milvus::knowhere::IndexEnum::INDEX_RHNSWFlat, + knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, + knowhere::IndexEnum::INDEX_NSG, + knowhere::IndexEnum::INDEX_RHNSWFlat, }; return ret; } @@ -34,8 +34,8 @@ NM_List() { std::vector BIN_List() { static std::vector ret{ - milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, - milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, + knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, + knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, }; return ret; } @@ -43,8 +43,8 @@ BIN_List() { std::vector Need_ID_List() { static std::vector ret{ - // milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, - // milvus::knowhere::IndexEnum::INDEX_NSG, + // knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, + // knowhere::IndexEnum::INDEX_NSG, }; return ret; @@ -53,7 +53,7 @@ Need_ID_List() { std::vector Need_BuildAll_list() { static std::vector ret{ - milvus::knowhere::IndexEnum::INDEX_NSG, + knowhere::IndexEnum::INDEX_NSG, }; return ret; } @@ -74,27 +74,27 @@ is_in_list(const T& t, std::function()> list_func) { } bool -is_in_bin_list(const milvus::knowhere::IndexType& index_type) { +is_in_bin_list(const knowhere::IndexType& index_type) { return is_in_list(index_type, BIN_List); } bool -is_in_nm_list(const milvus::knowhere::IndexType& index_type) { +is_in_nm_list(const knowhere::IndexType& index_type) { return is_in_list(index_type, NM_List); } bool -is_in_need_build_all_list(const milvus::knowhere::IndexType& index_type) { +is_in_need_build_all_list(const knowhere::IndexType& index_type) { return is_in_list(index_type, Need_BuildAll_list); } bool -is_in_need_id_list(const milvus::knowhere::IndexType& index_type) { +is_in_need_id_list(const knowhere::IndexType& index_type) { return is_in_list(index_type, Need_ID_List); } bool -is_unsupported(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type) { +is_unsupported(const knowhere::IndexType& index_type, const knowhere::MetricType& metric_type) { return is_in_list>(std::make_tuple(index_type, metric_type), unsupported_index_combinations); } diff --git a/internal/core/src/indexbuilder/utils.h b/internal/core/src/indexbuilder/utils.h index ca59c14bcd..60f7c3879d 100644 --- a/internal/core/src/indexbuilder/utils.h +++ b/internal/core/src/indexbuilder/utils.h @@ -42,18 +42,18 @@ bool is_in_list(const T& t, std::function()> list_func); bool -is_in_bin_list(const milvus::knowhere::IndexType& index_type); +is_in_bin_list(const knowhere::IndexType& index_type); bool -is_in_nm_list(const milvus::knowhere::IndexType& index_type); +is_in_nm_list(const knowhere::IndexType& index_type); bool -is_in_need_build_all_list(const milvus::knowhere::IndexType& index_type); +is_in_need_build_all_list(const knowhere::IndexType& index_type); bool -is_in_need_id_list(const milvus::knowhere::IndexType& index_type); +is_in_need_id_list(const knowhere::IndexType& index_type); bool -is_unsupported(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type); +is_unsupported(const knowhere::IndexType& index_type, const knowhere::MetricType& metric_type); } // namespace milvus::indexbuilder diff --git a/internal/core/src/query/SearchBruteForce.cpp b/internal/core/src/query/SearchBruteForce.cpp index efb29a71f9..92b45113b6 100644 --- a/internal/core/src/query/SearchBruteForce.cpp +++ b/internal/core/src/query/SearchBruteForce.cpp @@ -124,7 +124,7 @@ FloatSearchBruteForce(const dataset::SearchDataset& dataset, #elif defined(__linux__) || defined(__MINGW64__) if (metric_type == MetricType::METRIC_L2) { faiss::float_maxheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_ids(), sub_qr.get_distances()}; - faiss::knn_L2sqr(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset); + faiss::knn_L2sqr(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, nullptr, bitset); } else { faiss::float_minheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_ids(), sub_qr.get_distances()}; faiss::knn_inner_product(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset); diff --git a/internal/core/src/query/SearchOnIndex.cpp b/internal/core/src/query/SearchOnIndex.cpp index 2e1e3aeb29..e8e03b9799 100644 --- a/internal/core/src/query/SearchOnIndex.cpp +++ b/internal/core/src/query/SearchOnIndex.cpp @@ -30,8 +30,8 @@ SearchOnIndex(const dataset::SearchDataset& search_dataset, auto& indexing_nonconst = const_cast(indexing); auto ans = indexing_nonconst.Query(dataset, search_conf, bitset); - auto dis = ans->Get(milvus::knowhere::meta::DISTANCE); - auto uids = ans->Get(milvus::knowhere::meta::IDS); + auto dis = ans->Get(knowhere::meta::DISTANCE); + auto uids = ans->Get(knowhere::meta::IDS); SubSearchResult sub_qr(num_queries, topK, metric_type, round_decimal); std::copy_n(dis, num_queries * topK, sub_qr.get_distances()); diff --git a/internal/core/src/query/SearchOnSealed.cpp b/internal/core/src/query/SearchOnSealed.cpp index 006908a6be..0dcdeffc5b 100644 --- a/internal/core/src/query/SearchOnSealed.cpp +++ b/internal/core/src/query/SearchOnSealed.cpp @@ -83,10 +83,10 @@ SearchOnSealed(const Schema& schema, auto ds = knowhere::GenDataset(num_queries, dim, query_data); auto conf = search_info.search_params_; - conf[milvus::knowhere::meta::TOPK] = search_info.topk_; - conf[milvus::knowhere::Metric::TYPE] = MetricTypeToName(field_indexing->metric_type_); + conf[knowhere::meta::TOPK] = search_info.topk_; + conf[knowhere::Metric::TYPE] = MetricTypeToName(field_indexing->metric_type_); auto index_type = field_indexing->indexing_->index_type(); - auto adapter = milvus::knowhere::AdapterMgr::GetInstance().GetAdapter(index_type); + auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type); AssertInfo(adapter->CheckSearch(conf, index_type, field_indexing->indexing_->index_mode()), "[SearchOnSealed]Search params check failed"); return field_indexing->indexing_->Query(ds, conf, bitset); diff --git a/internal/core/src/segcore/load_index_c.cpp b/internal/core/src/segcore/load_index_c.cpp index bdca33ca27..b8f7034338 100644 --- a/internal/core/src/segcore/load_index_c.cpp +++ b/internal/core/src/segcore/load_index_c.cpp @@ -80,20 +80,19 @@ CStatus AppendIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) { try { auto load_index_info = (LoadIndexInfo*)c_load_index_info; - auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; + auto binary_set = (knowhere::BinarySet*)c_binary_set; auto& index_params = load_index_info->index_params; bool find_index_type = index_params.count("index_type") > 0 ? true : false; bool find_index_mode = index_params.count("index_mode") > 0 ? true : false; AssertInfo(find_index_type == true, "Can't find index type in index_params"); - milvus::knowhere::IndexMode mode; + knowhere::IndexMode mode; if (find_index_mode) { - mode = index_params["index_mode"] == "CPU" ? milvus::knowhere::IndexMode::MODE_CPU - : milvus::knowhere::IndexMode::MODE_GPU; + mode = index_params["index_mode"] == "CPU" ? knowhere::IndexMode::MODE_CPU : knowhere::IndexMode::MODE_GPU; } else { - mode = milvus::knowhere::IndexMode::MODE_CPU; + mode = knowhere::IndexMode::MODE_CPU; } load_index_info->index = - milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_params["index_type"], mode); + knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_params["index_type"], mode); load_index_info->index->Load(*binary_set); auto status = CStatus(); status.error_code = Success; diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt index 04f2f45826..cd7c384b3e 100644 --- a/internal/core/thirdparty/knowhere/CMakeLists.txt +++ b/internal/core/thirdparty/knowhere/CMakeLists.txt @@ -11,8 +11,8 @@ # or implied. See the License for the specific language governing permissions and limitations under the License. #------------------------------------------------------------------------------- -set( KNOWHERE_VERSION v1.0.1 ) -set( KNOWHERE_SOURCE_MD5 "0ac8a0802f94c2370f13a40527c57ced" ) +set( KNOWHERE_VERSION v1.1.0 ) +set( KNOWHERE_SOURCE_MD5 "9c326d3ba94184f4f646903f1ad36d49" ) if ( DEFINED ENV{MILVUS_KNOWHERE_URL} ) set( KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}" ) @@ -34,7 +34,7 @@ macro(build_knowhere) set(KNOWHERE_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_BUILD_TYPE={BUILD_TYPE} + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${KNOWHERE_INSTALL_PREFIX} ) diff --git a/internal/core/unittest/test_c_api.cpp b/internal/core/unittest/test_c_api.cpp index 17f25fdf9a..3abd0be491 100644 --- a/internal/core/unittest/test_c_api.cpp +++ b/internal/core/unittest/test_c_api.cpp @@ -34,7 +34,7 @@ namespace chrono = std::chrono; using namespace milvus; using namespace milvus::segcore; -using namespace milvus::knowhere; +using namespace knowhere; namespace { const int DIM = 16; @@ -187,10 +187,10 @@ generate_collection_schema(std::string metric_type, int dim, bool is_binary) { VecIndexPtr generate_index( - void* raw_data, milvus::knowhere::Config conf, int64_t dim, int64_t topK, int64_t N, std::string index_type) { + void* raw_data, knowhere::Config conf, int64_t dim, int64_t topK, int64_t N, std::string index_type) { auto indexing = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type, knowhere::IndexMode::MODE_CPU); - auto database = milvus::knowhere::GenDataset(N, dim, raw_data); + auto database = knowhere::GenDataset(N, dim, raw_data); indexing->Train(database, conf); indexing->AddWithoutIds(database, conf); EXPECT_EQ(indexing->Count(), N); @@ -1005,17 +1005,17 @@ TEST(CApiTest, LoadIndexInfo) { auto N = 1024 * 10; auto [raw_data, timestamps, uids] = generate_data(N); - auto indexing = std::make_shared(); - auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 4}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, - {milvus::knowhere::meta::DEVICEID, 0}}; + auto indexing = std::make_shared(); + auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::nprobe, 4}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, 0}}; - auto database = milvus::knowhere::GenDataset(N, DIM, raw_data.data()); + auto database = knowhere::GenDataset(N, DIM, raw_data.data()); indexing->Train(database, conf); indexing->AddWithoutIds(database, conf); EXPECT_EQ(indexing->Count(), N); @@ -1048,17 +1048,17 @@ TEST(CApiTest, LoadIndex_Search) { auto N = 1024 * 1024; auto num_query = 100; auto [raw_data, timestamps, uids] = generate_data(N); - auto indexing = std::make_shared(); - auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 4}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, - {milvus::knowhere::meta::DEVICEID, 0}}; + auto indexing = std::make_shared(); + auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::nprobe, 4}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, 0}}; - auto database = milvus::knowhere::GenDataset(N, DIM, raw_data.data()); + auto database = knowhere::GenDataset(N, DIM, raw_data.data()); indexing->Train(database, conf); indexing->AddWithoutIds(database, conf); @@ -1073,18 +1073,18 @@ TEST(CApiTest, LoadIndex_Search) { auto& index_params = load_index_info.index_params; index_params["index_type"] = "IVF_PQ"; index_params["index_mode"] = "CPU"; - auto mode = milvus::knowhere::IndexMode::MODE_CPU; + auto mode = knowhere::IndexMode::MODE_CPU; load_index_info.index = - milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_params["index_type"], mode); + knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_params["index_type"], mode); load_index_info.index->Load(binary_set); // search - auto query_dataset = milvus::knowhere::GenDataset(num_query, DIM, raw_data.data() + DIM * 4200); + auto query_dataset = knowhere::GenDataset(num_query, DIM, raw_data.data() + DIM * 4200); auto result = indexing->Query(query_dataset, conf, nullptr); - auto ids = result->Get(milvus::knowhere::meta::IDS); - auto dis = result->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result->Get(knowhere::meta::IDS); + auto dis = result->Get(knowhere::meta::DISTANCE); // for (int i = 0; i < std::min(num_query * K, 100); ++i) { // std::cout << ids[i] << "->" << dis[i] << std::endl; //} @@ -1150,21 +1150,21 @@ TEST(CApiTest, Indexing_Without_Predicate) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 10}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, - {milvus::knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::nprobe, 10}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, 0}}; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset - auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); - auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result_on_index->Get(knowhere::meta::IDS); + auto dis = result_on_index->Get(knowhere::meta::DISTANCE); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -1269,21 +1269,21 @@ TEST(CApiTest, Indexing_Expr_Without_Predicate) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 10}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, - {milvus::knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::nprobe, 10}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, 0}}; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset - auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); - auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result_on_index->Get(knowhere::meta::IDS); + auto dis = result_on_index->Get(knowhere::meta::DISTANCE); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -1405,22 +1405,22 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 10}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, - {milvus::knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::nprobe, 10}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, 0}}; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset - auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); - auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result_on_index->Get(knowhere::meta::IDS); + auto dis = result_on_index->Get(knowhere::meta::DISTANCE); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -1556,22 +1556,22 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 10}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, - {milvus::knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::nprobe, 10}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, 0}}; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset - auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); - auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result_on_index->Get(knowhere::meta::IDS); + auto dis = result_on_index->Get(knowhere::meta::DISTANCE); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -1691,22 +1691,22 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 10}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, - {milvus::knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::nprobe, 10}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, 0}}; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset - auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); - auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result_on_index->Get(knowhere::meta::IDS); + auto dis = result_on_index->Get(knowhere::meta::DISTANCE); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -1835,22 +1835,22 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Term) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 10}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, - {milvus::knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::nprobe, 10}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, 0}}; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset - auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); - auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result_on_index->Get(knowhere::meta::IDS); + auto dis = result_on_index->Get(knowhere::meta::DISTANCE); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -1971,23 +1971,23 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nprobe, 10}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::JACCARD}, + auto conf = knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nprobe, 10}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::JACCARD}, }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT); // gen query dataset - auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); - auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result_on_index->Get(knowhere::meta::IDS); + auto dis = result_on_index->Get(knowhere::meta::DISTANCE); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2121,23 +2121,23 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) { ASSERT_TRUE(res_before_load_index.error_code == Success) << res_before_load_index.error_msg; // load index to segment - auto conf = milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nprobe, 10}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::JACCARD}, + auto conf = knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nprobe, 10}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::JACCARD}, }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT); // gen query dataset - auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); - auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result_on_index->Get(knowhere::meta::IDS); + auto dis = result_on_index->Get(knowhere::meta::DISTANCE); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2257,23 +2257,23 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nprobe, 10}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::JACCARD}, + auto conf = knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nprobe, 10}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::JACCARD}, }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT); // gen query dataset - auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); - auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result_on_index->Get(knowhere::meta::IDS); + auto dis = result_on_index->Get(knowhere::meta::DISTANCE); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2406,23 +2406,23 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nprobe, 10}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::JACCARD}, + auto conf = knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nprobe, 10}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::JACCARD}, }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT); // gen query dataset - auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); - auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result_on_index->Get(knowhere::meta::IDS); + auto dis = result_on_index->Get(knowhere::meta::DISTANCE); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2585,22 +2585,22 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) { Timestamp time = 10000000; // load index to segment - auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 10}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, - {milvus::knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::nprobe, 10}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, 0}}; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset - auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); - auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result_on_index->Get(knowhere::meta::IDS); + auto dis = result_on_index->Get(knowhere::meta::DISTANCE); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2625,11 +2625,11 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) { AppendIndex(c_load_index_info, (CBinarySet)&binary_set); auto load_index_info = (LoadIndexInfo*)c_load_index_info; - auto query_dataset2 = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset2 = knowhere::GenDataset(num_queries, DIM, query_ptr); auto fuck2 = load_index_info->index; auto result_on_index2 = fuck2->Query(query_dataset2, conf, nullptr); - auto ids2 = result_on_index2->Get(milvus::knowhere::meta::IDS); - auto dis2 = result_on_index2->Get(milvus::knowhere::meta::DISTANCE); + auto ids2 = result_on_index2->Get(knowhere::meta::IDS); + auto dis2 = result_on_index2->Get(knowhere::meta::DISTANCE); int i = 1 + 1; ++i; @@ -2844,22 +2844,22 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) { Timestamp time = 10000000; // load index to segment - auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, TOPK}, - {milvus::knowhere::IndexParams::nlist, 100}, - {milvus::knowhere::IndexParams::nprobe, 10}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, - {milvus::knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::IndexParams::nlist, 100}, + {knowhere::IndexParams::nprobe, 10}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, + {knowhere::meta::DEVICEID, 0}}; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset - auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(milvus::knowhere::meta::IDS); - auto dis = result_on_index->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result_on_index->Get(knowhere::meta::IDS); + auto dis = result_on_index->Get(knowhere::meta::DISTANCE); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2884,11 +2884,11 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) { AppendIndex(c_load_index_info, (CBinarySet)&binary_set); auto load_index_info = (LoadIndexInfo*)c_load_index_info; - auto query_dataset2 = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr); + auto query_dataset2 = knowhere::GenDataset(num_queries, DIM, query_ptr); auto fuck2 = load_index_info->index; auto result_on_index2 = fuck2->Query(query_dataset2, conf, nullptr); - auto ids2 = result_on_index2->Get(milvus::knowhere::meta::IDS); - auto dis2 = result_on_index2->Get(milvus::knowhere::meta::DISTANCE); + auto ids2 = result_on_index2->Get(knowhere::meta::IDS); + auto dis2 = result_on_index2->Get(knowhere::meta::DISTANCE); int i = 1 + 1; ++i; diff --git a/internal/core/unittest/test_conf_adapter_mgr.cpp b/internal/core/unittest/test_conf_adapter_mgr.cpp index c840c8e903..b6eb132e71 100644 --- a/internal/core/unittest/test_conf_adapter_mgr.cpp +++ b/internal/core/unittest/test_conf_adapter_mgr.cpp @@ -16,8 +16,8 @@ TEST(AdapterMgr, MultiThread) { auto run_case = [&]() { - auto& ins = milvus::knowhere::AdapterMgr::GetInstance(); - auto adapter = ins.GetAdapter(milvus::knowhere::IndexEnum::INDEX_HNSW); + auto& ins = knowhere::AdapterMgr::GetInstance(); + auto adapter = ins.GetAdapter(knowhere::IndexEnum::INDEX_HNSW); ASSERT_TRUE(adapter != nullptr); ASSERT_ANY_THROW(ins.GetAdapter("not supported now!")); }; diff --git a/internal/core/unittest/test_index_c_api.cpp b/internal/core/unittest/test_index_c_api.cpp index ee512817fc..032965773a 100644 --- a/internal/core/unittest/test_index_c_api.cpp +++ b/internal/core/unittest/test_index_c_api.cpp @@ -31,8 +31,8 @@ constexpr int NB = 10; TEST(FloatVecIndex, All) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ; - auto metric_type = milvus::knowhere::Metric::L2; + auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFPQ; + auto metric_type = knowhere::Metric::L2; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -82,8 +82,8 @@ TEST(FloatVecIndex, All) { } TEST(BinaryVecIndex, All) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT; - auto metric_type = milvus::knowhere::Metric::JACCARD; + auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT; + auto metric_type = knowhere::Metric::JACCARD; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -134,7 +134,7 @@ TEST(BinaryVecIndex, All) { TEST(CBoolIndexTest, All) { schemapb::BoolArray half; - milvus::knowhere::DatasetPtr half_ds; + knowhere::DatasetPtr half_ds; for (size_t i = 0; i < NB; i++) { *(half.mutable_data()->Add()) = (i % 2) == 0; @@ -159,8 +159,8 @@ TEST(CBoolIndexTest, All) { ASSERT_EQ(Success, status.error_code); } { - status = BuildScalarIndex(index, half_ds->Get(milvus::knowhere::meta::ROWS), - half_ds->Get(milvus::knowhere::meta::TENSOR)); + status = BuildScalarIndex(index, half_ds->Get(knowhere::meta::ROWS), + half_ds->Get(knowhere::meta::TENSOR)); ASSERT_EQ(Success, status.error_code); } { @@ -185,7 +185,7 @@ TEST(CBoolIndexTest, All) { } } - delete[](char*) half_ds->Get(milvus::knowhere::meta::TENSOR); + delete[](char*) half_ds->Get(knowhere::meta::TENSOR); } // TODO: more scalar type. @@ -260,8 +260,8 @@ TEST(CStringIndexTest, All) { ASSERT_EQ(Success, status.error_code); } { - status = BuildScalarIndex(index, str_ds->Get(milvus::knowhere::meta::ROWS), - str_ds->Get(milvus::knowhere::meta::TENSOR)); + status = BuildScalarIndex(index, str_ds->Get(knowhere::meta::ROWS), + str_ds->Get(knowhere::meta::TENSOR)); ASSERT_EQ(Success, status.error_code); } { @@ -286,5 +286,5 @@ TEST(CStringIndexTest, All) { } } - delete[](char*) str_ds->Get(milvus::knowhere::meta::TENSOR); + delete[](char*) str_ds->Get(knowhere::meta::TENSOR); } diff --git a/internal/core/unittest/test_index_wrapper.cpp b/internal/core/unittest/test_index_wrapper.cpp index 5101c49697..e4dd673325 100644 --- a/internal/core/unittest/test_index_wrapper.cpp +++ b/internal/core/unittest/test_index_wrapper.cpp @@ -28,13 +28,13 @@ constexpr int64_t NB = 1000; namespace indexcgo = milvus::proto::indexcgo; -using Param = std::pair; +using Param = std::pair; class IndexWrapperTest : public ::testing::TestWithParam { protected: void SetUp() override { - milvus::engine::KnowhereConfig::SetStatisticsLevel(3); + knowhere::KnowhereConfig::SetStatisticsLevel(3); auto param = GetParam(); index_type = param.first; @@ -42,24 +42,28 @@ class IndexWrapperTest : public ::testing::TestWithParam { std::tie(type_params, index_params) = generate_params(index_type, metric_type); std::map is_binary_map = { - {milvus::knowhere::IndexEnum::INDEX_FAISS_IDMAP, false}, - {milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, false}, - {milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, false}, - {milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, false}, - {milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, true}, - {milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, true}, + {knowhere::IndexEnum::INDEX_FAISS_IDMAP, false}, + {knowhere::IndexEnum::INDEX_FAISS_IVFPQ, false}, + {knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, false}, + {knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, false}, + {knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, true}, + {knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, true}, #ifdef MILVUS_SUPPORT_SPTAG - {milvus::knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, false}, - {milvus::knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, false}, + {knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, false}, + {knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, false}, +#endif + {knowhere::IndexEnum::INDEX_HNSW, false}, + {knowhere::IndexEnum::INDEX_ANNOY, false}, + {knowhere::IndexEnum::INDEX_RHNSWFlat, false}, + {knowhere::IndexEnum::INDEX_RHNSWPQ, false}, + {knowhere::IndexEnum::INDEX_RHNSWSQ, false}, +#ifdef MILVUS_SUPPORT_NGT + {knowhere::IndexEnum::INDEX_NGTPANNG, false}, + {knowhere::IndexEnum::INDEX_NGTONNG, false}, +#endif +#ifdef MILVUS_SUPPORT_NSG + {knowhere::IndexEnum::INDEX_NSG, false}, #endif - {milvus::knowhere::IndexEnum::INDEX_HNSW, false}, - {milvus::knowhere::IndexEnum::INDEX_ANNOY, false}, - {milvus::knowhere::IndexEnum::INDEX_RHNSWFlat, false}, - {milvus::knowhere::IndexEnum::INDEX_RHNSWPQ, false}, - {milvus::knowhere::IndexEnum::INDEX_RHNSWSQ, false}, - {milvus::knowhere::IndexEnum::INDEX_NGTPANNG, false}, - {milvus::knowhere::IndexEnum::INDEX_NGTONNG, false}, - {milvus::knowhere::IndexEnum::INDEX_NSG, false}, }; is_binary = is_binary_map[index_type]; @@ -73,14 +77,14 @@ class IndexWrapperTest : public ::testing::TestWithParam { auto dataset = GenDataset(NB, metric_type, is_binary); if (!is_binary) { xb_data = dataset.get_col(0); - xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); xq_data = dataset.get_col(0); - xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data()); + xq_dataset = knowhere::GenDataset(NQ, DIM, xq_data.data()); } else { xb_bin_data = dataset.get_col(0); - xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_bin_data.data()); + xb_dataset = knowhere::GenDataset(NB, DIM, xb_bin_data.data()); xq_bin_data = dataset.get_col(0); - xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_bin_data.data()); + xq_dataset = knowhere::GenDataset(NQ, DIM, xq_bin_data.data()); } } @@ -94,59 +98,59 @@ class IndexWrapperTest : public ::testing::TestWithParam { indexcgo::IndexParams index_params; std::string type_params_str, index_params_str; bool is_binary; - milvus::knowhere::DatasetPtr xb_dataset; + knowhere::DatasetPtr xb_dataset; std::vector xb_data; std::vector xb_bin_data; - std::vector ids; - milvus::knowhere::DatasetPtr xq_dataset; + std::vector ids; + knowhere::DatasetPtr xq_dataset; std::vector xq_data; std::vector xq_bin_data; }; TEST(PQ, Build) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ; - auto metric_type = milvus::knowhere::Metric::L2; + auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFPQ; + auto metric_type = knowhere::Metric::L2; auto conf = generate_conf(index_type, metric_type); - auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); + auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto dataset = GenDataset(NB, metric_type, false); auto xb_data = dataset.get_col(0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); ASSERT_NO_THROW(index->Train(xb_dataset, conf)); ASSERT_NO_THROW(index->AddWithoutIds(xb_dataset, conf)); } TEST(IVFFLATNM, Build) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; - auto metric_type = milvus::knowhere::Metric::L2; + auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; + auto metric_type = knowhere::Metric::L2; auto conf = generate_conf(index_type, metric_type); - auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); + auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto dataset = GenDataset(NB, metric_type, false); auto xb_data = dataset.get_col(0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); ASSERT_NO_THROW(index->Train(xb_dataset, conf)); ASSERT_NO_THROW(index->AddWithoutIds(xb_dataset, conf)); } TEST(IVFFLATNM, Query) { - milvus::engine::KnowhereConfig::SetStatisticsLevel(3); + knowhere::KnowhereConfig::SetStatisticsLevel(3); - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; - auto metric_type = milvus::knowhere::Metric::L2; + auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; + auto metric_type = knowhere::Metric::L2; auto conf = generate_conf(index_type, metric_type); - auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); + auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto dataset = GenDataset(NB, metric_type, false); auto xb_data = dataset.get_col(0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); ASSERT_NO_THROW(index->Train(xb_dataset, conf)); ASSERT_NO_THROW(index->AddWithoutIds(xb_dataset, conf)); auto bs = index->Serialize(conf); - auto bptr = std::make_shared(); + auto bptr = std::make_shared(); bptr->data = std::shared_ptr((uint8_t*)xb_data.data(), [&](uint8_t*) {}); bptr->size = DIM * NB * sizeof(float); bs.Append(RAW_DATA, bptr); index->Load(bs); auto xq_data = dataset.get_col(0); - auto xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data()); + auto xq_dataset = knowhere::GenDataset(NQ, DIM, xq_data.data()); auto result = index->Query(xq_dataset, conf, nullptr); index->UpdateIndexSize(); @@ -157,36 +161,38 @@ TEST(IVFFLATNM, Query) { index->ClearStatistics(); } +#ifdef MILVUS_SUPPORT_NSG TEST(NSG, Query) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_NSG; - auto metric_type = milvus::knowhere::Metric::L2; + auto index_type = knowhere::IndexEnum::INDEX_NSG; + auto metric_type = knowhere::Metric::L2; auto conf = generate_conf(index_type, metric_type); - auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); + auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto dataset = GenDataset(NB, metric_type, false); auto xb_data = dataset.get_col(0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); index->BuildAll(xb_dataset, conf); auto bs = index->Serialize(conf); - auto bptr = std::make_shared(); + auto bptr = std::make_shared(); bptr->data = std::shared_ptr((uint8_t*)xb_data.data(), [&](uint8_t*) {}); bptr->size = DIM * NB * sizeof(float); bs.Append(RAW_DATA, bptr); index->Load(bs); auto xq_data = dataset.get_col(0); - auto xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data()); + auto xq_dataset = knowhere::GenDataset(NQ, DIM, xq_data.data()); auto result = index->Query(xq_dataset, conf, nullptr); } +#endif TEST(BINFLAT, Build) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT; - auto metric_type = milvus::knowhere::Metric::JACCARD; + auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT; + auto metric_type = knowhere::Metric::JACCARD; auto conf = generate_conf(index_type, metric_type); - auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); + auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto dataset = GenDataset(NB, metric_type, true); auto xb_data = dataset.get_col(0); - std::vector ids(NB, 0); + std::vector ids(NB, 0); std::iota(ids.begin(), ids.end(), 0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); ASSERT_NO_THROW(index->BuildAll(xb_dataset, conf)); } @@ -203,30 +209,30 @@ print_query_result(const std::unique_ptr(0); - std::vector ids(nb, 0); + std::vector ids(nb, 0); std::iota(ids.begin(), ids.end(), 0); - auto xb_dataset = milvus::knowhere::GenDataset(nb, dim, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(nb, dim, xb_data.data()); index->BuildAll(xb_dataset, conf); auto xq_data = dataset.get_col(0); - auto xq_dataset = milvus::knowhere::GenDataset(nq, dim, xq_data.data()); + auto xq_dataset = knowhere::GenDataset(nq, dim, xq_data.data()); auto result = index->Query(xq_dataset, conf, nullptr); - auto hit_ids = result->Get(milvus::knowhere::meta::IDS); - auto distances = result->Get(milvus::knowhere::meta::DISTANCE); + auto hit_ids = result->Get(knowhere::meta::IDS); + auto distances = result->Get(knowhere::meta::DISTANCE); auto query_res = std::make_unique(); query_res->nq = nq; @@ -247,21 +253,21 @@ TEST(BinIVFFlat, Build_and_Query) { } TEST(BINIDMAP, Build) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP; - auto metric_type = milvus::knowhere::Metric::JACCARD; + auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP; + auto metric_type = knowhere::Metric::JACCARD; auto conf = generate_conf(index_type, metric_type); - auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); + auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto dataset = GenDataset(NB, metric_type, true); auto xb_data = dataset.get_col(0); - std::vector ids(NB, 0); + std::vector ids(NB, 0); std::iota(ids.begin(), ids.end(), 0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); ASSERT_NO_THROW(index->BuildAll(xb_dataset, conf)); } TEST(PQWrapper, Build) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ; - auto metric_type = milvus::knowhere::Metric::L2; + auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFPQ; + auto metric_type = knowhere::Metric::L2; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -273,15 +279,15 @@ TEST(PQWrapper, Build) { assert(ok); auto dataset = GenDataset(NB, metric_type, false); auto xb_data = dataset.get_col(0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); auto index = std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); } TEST(IVFFLATNMWrapper, Build) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; - auto metric_type = milvus::knowhere::Metric::L2; + auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; + auto metric_type = knowhere::Metric::L2; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -293,7 +299,7 @@ TEST(IVFFLATNMWrapper, Build) { assert(ok); auto dataset = GenDataset(NB, metric_type, false); auto xb_data = dataset.get_col(0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); auto index = std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); @@ -301,8 +307,8 @@ TEST(IVFFLATNMWrapper, Build) { TEST(IVFFLATNMWrapper, Codec) { int64_t flat_nb = 100000; - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; - auto metric_type = milvus::knowhere::Metric::L2; + auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; + auto metric_type = knowhere::Metric::L2; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -314,7 +320,7 @@ TEST(IVFFLATNMWrapper, Codec) { assert(ok); auto dataset = GenDataset(flat_nb, metric_type, false); auto xb_data = dataset.get_col(0); - auto xb_dataset = milvus::knowhere::GenDataset(flat_nb, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(flat_nb, DIM, xb_data.data()); auto index_wrapper = std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_NO_THROW(index_wrapper->BuildWithoutIds(xb_dataset)); @@ -335,8 +341,8 @@ TEST(IVFFLATNMWrapper, Codec) { } TEST(BinFlatWrapper, Build) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT; - auto metric_type = milvus::knowhere::Metric::JACCARD; + auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT; + auto metric_type = knowhere::Metric::JACCARD; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -348,9 +354,9 @@ TEST(BinFlatWrapper, Build) { assert(ok); auto dataset = GenDataset(NB, metric_type, true); auto xb_data = dataset.get_col(0); - std::vector ids(NB, 0); + std::vector ids(NB, 0); std::iota(ids.begin(), ids.end(), 0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); auto index = std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); @@ -358,8 +364,8 @@ TEST(BinFlatWrapper, Build) { } TEST(BinIdMapWrapper, Build) { - auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP; - auto metric_type = milvus::knowhere::Metric::JACCARD; + auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP; + auto metric_type = knowhere::Metric::JACCARD; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -371,9 +377,9 @@ TEST(BinIdMapWrapper, Build) { assert(ok); auto dataset = GenDataset(NB, metric_type, true); auto xb_data = dataset.get_col(0); - std::vector ids(NB, 0); + std::vector ids(NB, 0); std::iota(ids.begin(), ids.end(), 0); - auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); + auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data()); auto index = std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); @@ -384,25 +390,30 @@ INSTANTIATE_TEST_CASE_P( IndexTypeParameters, IndexWrapperTest, ::testing::Values( - std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_IDMAP, milvus::knowhere::Metric::L2), - std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, milvus::knowhere::Metric::L2), - std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, milvus::knowhere::Metric::L2), - std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, milvus::knowhere::Metric::L2), - std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, milvus::knowhere::Metric::JACCARD), - std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, milvus::knowhere::Metric::TANIMOTO), - std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, milvus::knowhere::Metric::JACCARD), + std::pair(knowhere::IndexEnum::INDEX_FAISS_IDMAP, knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFPQ, knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::Metric::JACCARD), + std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::Metric::TANIMOTO), + std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, knowhere::Metric::JACCARD), #ifdef MILVUS_SUPPORT_SPTAG - std::pair(milvus::knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, milvus::knowhere::Metric::L2), - std::pair(milvus::knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, milvus::knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, knowhere::Metric::L2), #endif - std::pair(milvus::knowhere::IndexEnum::INDEX_HNSW, milvus::knowhere::Metric::L2), - std::pair(milvus::knowhere::IndexEnum::INDEX_ANNOY, milvus::knowhere::Metric::L2), - std::pair(milvus::knowhere::IndexEnum::INDEX_RHNSWFlat, milvus::knowhere::Metric::L2), - std::pair(milvus::knowhere::IndexEnum::INDEX_RHNSWPQ, milvus::knowhere::Metric::L2), - std::pair(milvus::knowhere::IndexEnum::INDEX_RHNSWSQ, milvus::knowhere::Metric::L2), - std::pair(milvus::knowhere::IndexEnum::INDEX_NGTPANNG, milvus::knowhere::Metric::L2), - std::pair(milvus::knowhere::IndexEnum::INDEX_NGTONNG, milvus::knowhere::Metric::L2), - std::pair(milvus::knowhere::IndexEnum::INDEX_NSG, milvus::knowhere::Metric::L2))); + std::pair(knowhere::IndexEnum::INDEX_HNSW, knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_ANNOY, knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_RHNSWFlat, knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_RHNSWPQ, knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_RHNSWSQ, knowhere::Metric::L2) +#ifdef MILVUS_SUPPORT_NGT + std::pair(knowhere::IndexEnum::INDEX_NGTPANNG, knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_NGTONNG, knowhere::Metric::L2), +#endif +#ifdef MILVUS_SUPPORT_NSG + std::pair(knowhere::IndexEnum::INDEX_NSG, knowhere::Metric::L2) +#endif + )); TEST_P(IndexWrapperTest, Constructor) { auto index = diff --git a/internal/core/unittest/test_indexing.cpp b/internal/core/unittest/test_indexing.cpp index 9c245a1f46..d950d02b59 100644 --- a/internal/core/unittest/test_indexing.cpp +++ b/internal/core/unittest/test_indexing.cpp @@ -111,14 +111,14 @@ TEST(Indexing, Naive) { auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(knowhere::IndexEnum::INDEX_FAISS_IVFPQ, knowhere::IndexMode::MODE_CPU); - auto conf = milvus::knowhere::Config{ + auto conf = knowhere::Config{ {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, TOPK}, {knowhere::IndexParams::nlist, 100}, {knowhere::IndexParams::nprobe, 4}, {knowhere::IndexParams::m, 4}, {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, {knowhere::meta::DEVICEID, 0}, }; @@ -193,7 +193,7 @@ TEST(Indexing, IVFFlat) { {knowhere::meta::TOPK, TOPK}, {knowhere::IndexParams::nlist, NLIST}, {knowhere::IndexParams::nprobe, NPROBE}, - {knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, {knowhere::meta::DEVICEID, 0}}; auto database = knowhere::GenDataset(N, DIM, raw_data.data()); @@ -210,8 +210,8 @@ TEST(Indexing, IVFFlat) { auto result = indexing->Query(dataset, conf, nullptr); std::cout << "query ivf " << timer.get_step_seconds() << " seconds" << std::endl; - auto ids = result->Get(milvus::knowhere::meta::IDS); - auto dis = result->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result->Get(knowhere::meta::IDS); + auto dis = result->Get(knowhere::meta::DISTANCE); for (int i = 0; i < std::min(NQ * TOPK, 100); ++i) { std::cout << ids[i] << "->" << dis[i] << std::endl; } @@ -233,7 +233,7 @@ TEST(Indexing, IVFFlatNM) { {knowhere::meta::TOPK, TOPK}, {knowhere::IndexParams::nlist, NLIST}, {knowhere::IndexParams::nprobe, NPROBE}, - {knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, {knowhere::meta::DEVICEID, 0}}; auto database = knowhere::GenDataset(N, DIM, raw_data.data()); @@ -244,9 +244,9 @@ TEST(Indexing, IVFFlatNM) { std::cout << "insert ivf_nm " << timer.get_step_seconds() << " seconds" << std::endl; indexing->SetIndexSize(NQ * DIM * sizeof(float)); - milvus::knowhere::BinarySet bs = indexing->Serialize(conf); + knowhere::BinarySet bs = indexing->Serialize(conf); - milvus::knowhere::BinaryPtr bptr = std::make_shared(); + knowhere::BinaryPtr bptr = std::make_shared(); bptr->data = std::shared_ptr((uint8_t*)raw_data.data(), [&](uint8_t*) {}); bptr->size = DIM * N * sizeof(float); bs.Append(RAW_DATA, bptr); @@ -259,8 +259,8 @@ TEST(Indexing, IVFFlatNM) { auto result = indexing->Query(dataset, conf, nullptr); std::cout << "query ivf_nm " << timer.get_step_seconds() << " seconds" << std::endl; - auto ids = result->Get(milvus::knowhere::meta::IDS); - auto dis = result->Get(milvus::knowhere::meta::DISTANCE); + auto ids = result->Get(knowhere::meta::IDS); + auto dis = result->Get(knowhere::meta::DISTANCE); for (int i = 0; i < std::min(NQ * TOPK, 100); ++i) { std::cout << ids[i] << "->" << dis[i] << std::endl; } diff --git a/internal/core/unittest/test_scalar_index_creator.cpp b/internal/core/unittest/test_scalar_index_creator.cpp index ced848634f..c7940e671b 100644 --- a/internal/core/unittest/test_scalar_index_creator.cpp +++ b/internal/core/unittest/test_scalar_index_creator.cpp @@ -38,7 +38,7 @@ namespace indexcgo = milvus::proto::indexcgo; namespace schemapb = milvus::proto::schema; using milvus::indexbuilder::MapParams; using milvus::indexbuilder::ScalarIndexCreator; -using milvus::knowhere::scalar::OperatorType; +using knowhere::scalar::OperatorType; using ScalarTestParams = std::pair; namespace { @@ -225,7 +225,7 @@ TYPED_TEST_P(TypedScalarIndexCreatorTest, Codec) { std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); auto arr = GenArr(nb); const int64_t dim = 8; // not important here - auto dataset = milvus::knowhere::GenDataset(arr.size(), dim, arr.data()); + auto dataset = knowhere::GenDataset(arr.size(), dim, arr.data()); creator->Build(dataset); auto binary_set = creator->Serialize(); @@ -262,9 +262,9 @@ class BoolIndexTest : public ::testing::Test { void TearDown() override { - delete[](char*)(all_true_ds->Get(milvus::knowhere::meta::TENSOR)); - delete[](char*) all_false_ds->Get(milvus::knowhere::meta::TENSOR); - delete[](char*) half_ds->Get(milvus::knowhere::meta::TENSOR); + delete[](char*)(all_true_ds->Get(knowhere::meta::TENSOR)); + delete[](char*) all_false_ds->Get(knowhere::meta::TENSOR); + delete[](char*) half_ds->Get(knowhere::meta::TENSOR); } private: @@ -277,9 +277,9 @@ class BoolIndexTest : public ::testing::Test { schemapb::BoolArray all_true; schemapb::BoolArray all_false; schemapb::BoolArray half; - milvus::knowhere::DatasetPtr all_true_ds; - milvus::knowhere::DatasetPtr all_false_ds; - milvus::knowhere::DatasetPtr half_ds; + knowhere::DatasetPtr all_true_ds; + knowhere::DatasetPtr all_false_ds; + knowhere::DatasetPtr half_ds; size_t n; std::vector params; }; @@ -489,7 +489,7 @@ class StringIndexTest : public ::testing::Test { void TearDown() override { - delete[](char*)(str_ds->Get(milvus::knowhere::meta::TENSOR)); + delete[](char*)(str_ds->Get(knowhere::meta::TENSOR)); } private: @@ -501,7 +501,7 @@ class StringIndexTest : public ::testing::Test { protected: std::vector strs; schemapb::StringArray str_arr; - milvus::knowhere::DatasetPtr str_ds; + knowhere::DatasetPtr str_ds; std::vector params; }; diff --git a/internal/core/unittest/test_sealed.cpp b/internal/core/unittest/test_sealed.cpp index c68cb9aee4..dbda40b16e 100644 --- a/internal/core/unittest/test_sealed.cpp +++ b/internal/core/unittest/test_sealed.cpp @@ -80,7 +80,7 @@ TEST(Sealed, without_predicate) { {knowhere::meta::TOPK, topK}, {knowhere::IndexParams::nlist, 100}, {knowhere::IndexParams::nprobe, 10}, - {knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, {knowhere::meta::DEVICEID, 0}}; auto database = knowhere::GenDataset(N, dim, vec_col.data() + 1000 * dim); @@ -94,8 +94,8 @@ TEST(Sealed, without_predicate) { auto result = indexing->Query(query_dataset, conf, nullptr); - auto ids = result->Get(milvus::knowhere::meta::IDS); // for comparison - auto dis = result->Get(milvus::knowhere::meta::DISTANCE); // for comparison + auto ids = result->Get(knowhere::meta::IDS); // for comparison + auto dis = result->Get(knowhere::meta::DISTANCE); // for comparison std::vector vec_ids(ids, ids + topK * num_queries); std::vector vec_dis(dis, dis + topK * num_queries); @@ -180,7 +180,7 @@ TEST(Sealed, with_predicate) { {knowhere::meta::TOPK, topK}, {knowhere::IndexParams::nlist, 100}, {knowhere::IndexParams::nprobe, 10}, - {knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, {knowhere::meta::DEVICEID, 0}}; auto database = knowhere::GenDataset(N, dim, vec_col.data()); @@ -273,7 +273,7 @@ TEST(Sealed, LoadFieldData) { LoadIndexInfo vec_info; vec_info.field_id = fakevec_id.get(); vec_info.index = indexing; - vec_info.index_params["metric_type"] = milvus::knowhere::Metric::L2; + vec_info.index_params["metric_type"] = knowhere::Metric::L2; segment->LoadIndex(vec_info); ASSERT_EQ(segment->num_chunk(), 1); diff --git a/internal/core/unittest/test_utils/DataGen.h b/internal/core/unittest/test_utils/DataGen.h index 62ad27fce1..b39f14e7be 100644 --- a/internal/core/unittest/test_utils/DataGen.h +++ b/internal/core/unittest/test_utils/DataGen.h @@ -339,7 +339,7 @@ GenIndexing(int64_t N, int64_t dim, const float* vec) { // {knowhere::IndexParams::nprobe, 10}, auto conf = knowhere::Config{{knowhere::meta::DIM, dim}, {knowhere::IndexParams::nlist, 1024}, - {knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, + {knowhere::Metric::TYPE, knowhere::Metric::L2}, {knowhere::meta::DEVICEID, 0}}; auto database = knowhere::GenDataset(N, dim, vec); auto indexing = std::make_shared(); diff --git a/internal/core/unittest/test_utils/indexbuilder_test_utils.h b/internal/core/unittest/test_utils/indexbuilder_test_utils.h index 75f063b20f..f446b4579c 100644 --- a/internal/core/unittest/test_utils/indexbuilder_test_utils.h +++ b/internal/core/unittest/test_utils/indexbuilder_test_utils.h @@ -42,176 +42,176 @@ namespace indexcgo = milvus::proto::indexcgo; namespace schemapb = milvus::proto::schema; using milvus::indexbuilder::MapParams; using milvus::indexbuilder::ScalarIndexCreator; -using milvus::knowhere::scalar::OperatorType; +using knowhere::scalar::OperatorType; using ScalarTestParams = std::pair; namespace { auto -generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type) { - if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IDMAP) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, +generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& metric_type) { + if (index_type == knowhere::IndexEnum::INDEX_FAISS_IDMAP) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::IndexParams::nlist, 16}, - {milvus::knowhere::IndexParams::nprobe, 4}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::nlist, 16}, + {knowhere::IndexParams::nprobe, 4}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::IndexParams::nlist, 16}, - {milvus::knowhere::IndexParams::nprobe, 4}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::nlist, 16}, + {knowhere::IndexParams::nprobe, 4}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, #ifdef MILVUS_GPU_VERSION - {milvus::knowhere::meta::DEVICEID, DEVICEID}, + {knowhere::meta::DEVICEID, DEVICEID}, #endif }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::IndexParams::nlist, 16}, - {milvus::knowhere::IndexParams::nprobe, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::nlist, 16}, + {knowhere::IndexParams::nprobe, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, #ifdef MILVUS_GPU_VERSION - {milvus::knowhere::meta::DEVICEID, DEVICEID}, + {knowhere::meta::DEVICEID, DEVICEID}, #endif }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::IndexParams::nlist, 16}, - {milvus::knowhere::IndexParams::nprobe, 4}, - {milvus::knowhere::IndexParams::m, 4}, - {milvus::knowhere::IndexParams::nbits, 8}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::nlist, 16}, + {knowhere::IndexParams::nprobe, 4}, + {knowhere::IndexParams::m, 4}, + {knowhere::IndexParams::nbits, 8}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::Metric::TYPE, metric_type}, + } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::Metric::TYPE, metric_type}, }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_NSG) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::IndexParams::nlist, 163}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::IndexParams::nprobe, 8}, - {milvus::knowhere::IndexParams::knng, 20}, - {milvus::knowhere::IndexParams::search_length, 40}, - {milvus::knowhere::IndexParams::out_degree, 30}, - {milvus::knowhere::IndexParams::candidate, 100}, - {milvus::knowhere::Metric::TYPE, metric_type}, + } else if (index_type == knowhere::IndexEnum::INDEX_NSG) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::IndexParams::nlist, 163}, + {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::nprobe, 8}, + {knowhere::IndexParams::knng, 20}, + {knowhere::IndexParams::search_length, 40}, + {knowhere::IndexParams::out_degree, 30}, + {knowhere::IndexParams::candidate, 100}, + {knowhere::Metric::TYPE, metric_type}, }; #ifdef MILVUS_SUPPORT_SPTAG - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - // {milvus::knowhere::meta::TOPK, 10}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + } else if (index_type == knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + // {knowhere::meta::TOPK, 10}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - // {milvus::knowhere::meta::TOPK, 10}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + } else if (index_type == knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + // {knowhere::meta::TOPK, 10}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, }; #endif - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_HNSW) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::IndexParams::M, 16}, {milvus::knowhere::IndexParams::efConstruction, 200}, - {milvus::knowhere::IndexParams::ef, 200}, {milvus::knowhere::Metric::TYPE, metric_type}, + } else if (index_type == knowhere::IndexEnum::INDEX_HNSW) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::M, 16}, {knowhere::IndexParams::efConstruction, 200}, + {knowhere::IndexParams::ef, 200}, {knowhere::Metric::TYPE, metric_type}, }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_ANNOY) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::IndexParams::n_trees, 4}, - {milvus::knowhere::IndexParams::search_k, 100}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + } else if (index_type == knowhere::IndexEnum::INDEX_ANNOY) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::n_trees, 4}, + {knowhere::IndexParams::search_k, 100}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWFlat) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::IndexParams::M, 16}, - {milvus::knowhere::IndexParams::efConstruction, 200}, - {milvus::knowhere::IndexParams::ef, 200}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + } else if (index_type == knowhere::IndexEnum::INDEX_RHNSWFlat) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::M, 16}, + {knowhere::IndexParams::efConstruction, 200}, + {knowhere::IndexParams::ef, 200}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWPQ) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::IndexParams::M, 16}, - {milvus::knowhere::IndexParams::efConstruction, 200}, - {milvus::knowhere::IndexParams::ef, 200}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, - {milvus::knowhere::IndexParams::PQM, 8}, + } else if (index_type == knowhere::IndexEnum::INDEX_RHNSWPQ) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::M, 16}, + {knowhere::IndexParams::efConstruction, 200}, + {knowhere::IndexParams::ef, 200}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + {knowhere::IndexParams::PQM, 8}, }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWSQ) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::IndexParams::M, 16}, - {milvus::knowhere::IndexParams::efConstruction, 200}, - {milvus::knowhere::IndexParams::ef, 200}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + } else if (index_type == knowhere::IndexEnum::INDEX_RHNSWSQ) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::IndexParams::M, 16}, + {knowhere::IndexParams::efConstruction, 200}, + {knowhere::IndexParams::ef, 200}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_NGTPANNG) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::IndexParams::edge_size, 10}, - {milvus::knowhere::IndexParams::epsilon, 0.1}, - {milvus::knowhere::IndexParams::max_search_edges, 50}, - {milvus::knowhere::IndexParams::forcedly_pruned_edge_size, 60}, - {milvus::knowhere::IndexParams::selectively_pruned_edge_size, 30}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + } else if (index_type == knowhere::IndexEnum::INDEX_NGTPANNG) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::IndexParams::edge_size, 10}, + {knowhere::IndexParams::epsilon, 0.1}, + {knowhere::IndexParams::max_search_edges, 50}, + {knowhere::IndexParams::forcedly_pruned_edge_size, 60}, + {knowhere::IndexParams::selectively_pruned_edge_size, 30}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, }; - } else if (index_type == milvus::knowhere::IndexEnum::INDEX_NGTONNG) { - return milvus::knowhere::Config{ - {milvus::knowhere::meta::DIM, DIM}, - {milvus::knowhere::meta::TOPK, K}, - {milvus::knowhere::Metric::TYPE, metric_type}, - {milvus::knowhere::IndexParams::edge_size, 20}, - {milvus::knowhere::IndexParams::epsilon, 0.1}, - {milvus::knowhere::IndexParams::max_search_edges, 50}, - {milvus::knowhere::IndexParams::outgoing_edge_size, 5}, - {milvus::knowhere::IndexParams::incoming_edge_size, 40}, - {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, + } else if (index_type == knowhere::IndexEnum::INDEX_NGTONNG) { + return knowhere::Config{ + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::Metric::TYPE, metric_type}, + {knowhere::IndexParams::edge_size, 20}, + {knowhere::IndexParams::epsilon, 0.1}, + {knowhere::IndexParams::max_search_edges, 50}, + {knowhere::IndexParams::outgoing_edge_size, 5}, + {knowhere::IndexParams::incoming_edge_size, 40}, + {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4}, }; } - return milvus::knowhere::Config(); + return knowhere::Config(); } auto -generate_params(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type) { +generate_params(const knowhere::IndexType& index_type, const knowhere::MetricType& metric_type) { namespace indexcgo = milvus::proto::indexcgo; indexcgo::TypeParams type_params; @@ -233,9 +233,9 @@ generate_params(const milvus::knowhere::IndexType& index_type, const milvus::kno } auto -GenDataset(int64_t N, const milvus::knowhere::MetricType& metric_type, bool is_binary, int64_t dim = DIM) { +GenDataset(int64_t N, const knowhere::MetricType& metric_type, bool is_binary, int64_t dim = DIM) { auto schema = std::make_shared(); - auto faiss_metric_type = milvus::knowhere::GetMetricType(metric_type); + auto faiss_metric_type = knowhere::GetMetricType(metric_type); if (!is_binary) { schema->AddDebugField("fakevec", milvus::engine::DataType::VECTOR_FLOAT, dim, faiss_metric_type); return milvus::segcore::DataGen(schema, N); @@ -304,14 +304,14 @@ float CountDistance(const void* point_a, const void* point_b, int dim, - const milvus::knowhere::MetricType& metric, + const knowhere::MetricType& metric, bool is_binary = false) { if (point_a == nullptr || point_b == nullptr) { return std::numeric_limits::max(); } - if (metric == milvus::knowhere::Metric::L2) { + if (metric == knowhere::Metric::L2) { return L2(static_cast(point_a), static_cast(point_b), dim); - } else if (metric == milvus::knowhere::Metric::JACCARD) { + } else if (metric == knowhere::Metric::JACCARD) { return Jaccard(static_cast(point_a), static_cast(point_b), dim); } else { return std::numeric_limits::max(); @@ -320,13 +320,13 @@ CountDistance(const void* point_a, void CheckDistances(const QueryResultPtr& result, - const milvus::knowhere::DatasetPtr& base_dataset, - const milvus::knowhere::DatasetPtr& query_dataset, - const milvus::knowhere::MetricType& metric, + const knowhere::DatasetPtr& base_dataset, + const knowhere::DatasetPtr& query_dataset, + const knowhere::MetricType& metric, const float threshold = 1.0e-5) { - auto base_vecs = base_dataset->Get(milvus::knowhere::meta::TENSOR); - auto query_vecs = query_dataset->Get(milvus::knowhere::meta::TENSOR); - auto dim = base_dataset->Get(milvus::knowhere::meta::DIM); + auto base_vecs = base_dataset->Get(knowhere::meta::TENSOR); + auto query_vecs = query_dataset->Get(knowhere::meta::TENSOR); + auto dim = base_dataset->Get(knowhere::meta::DIM); auto nq = result->nq; auto k = result->topk; for (auto i = 0; i < nq; i++) { @@ -445,7 +445,7 @@ template inline void build_index(const std::unique_ptr>& creator, const std::vector& arr) { const int64_t dim = 8; // not important here - auto dataset = milvus::knowhere::GenDataset(arr.size(), dim, arr.data()); + auto dataset = knowhere::GenDataset(arr.size(), dim, arr.data()); creator->Build(dataset); } @@ -454,6 +454,6 @@ auto GenDsFromPB(const google::protobuf::Message& msg) { auto data = new char[msg.ByteSize()]; msg.SerializeToArray(data, msg.ByteSize()); - return milvus::knowhere::GenDataset(msg.ByteSize(), 8, data); + return knowhere::GenDataset(msg.ByteSize(), 8, data); } } // namespace diff --git a/internal/util/indexcgowrapper/index_test.go b/internal/util/indexcgowrapper/index_test.go index f29184c0bc..9d4039ee5c 100644 --- a/internal/util/indexcgowrapper/index_test.go +++ b/internal/util/indexcgowrapper/index_test.go @@ -72,8 +72,8 @@ func generateFloatVectorTestCases() []vecTestCase { {IndexFaissIVFSQ8, IP, false, schemapb.DataType_FloatVector}, //{IndexFaissIVFSQ8H, L2, false, schemapb.DataType_FloatVector}, // TODO: enable gpu //{IndexFaissIVFSQ8H, IP, false, schemapb.DataType_FloatVector}, - {IndexNsg, L2, false, schemapb.DataType_FloatVector}, - {IndexNsg, IP, false, schemapb.DataType_FloatVector}, + //{IndexNsg, L2, false, schemapb.DataType_FloatVector}, + //{IndexNsg, IP, false, schemapb.DataType_FloatVector}, {IndexHNSW, L2, false, schemapb.DataType_FloatVector}, {IndexHNSW, IP, false, schemapb.DataType_FloatVector}, {IndexRHNSWFlat, L2, false, schemapb.DataType_FloatVector}, @@ -84,10 +84,10 @@ func generateFloatVectorTestCases() []vecTestCase { {IndexRHNSWSQ, IP, false, schemapb.DataType_FloatVector}, {IndexANNOY, L2, false, schemapb.DataType_FloatVector}, {IndexANNOY, IP, false, schemapb.DataType_FloatVector}, - {IndexNGTPANNG, L2, false, schemapb.DataType_FloatVector}, - {IndexNGTPANNG, IP, false, schemapb.DataType_FloatVector}, - {IndexNGTONNG, L2, false, schemapb.DataType_FloatVector}, - {IndexNGTONNG, IP, false, schemapb.DataType_FloatVector}, + //{IndexNGTPANNG, L2, false, schemapb.DataType_FloatVector}, + //{IndexNGTPANNG, IP, false, schemapb.DataType_FloatVector}, + //{IndexNGTONNG, L2, false, schemapb.DataType_FloatVector}, + //{IndexNGTONNG, IP, false, schemapb.DataType_FloatVector}, } }