diff --git a/internal/core/src/common/LoadInfo.h b/internal/core/src/common/LoadInfo.h index 26cefcf34c..207ecc7936 100644 --- a/internal/core/src/common/LoadInfo.h +++ b/internal/core/src/common/LoadInfo.h @@ -20,7 +20,6 @@ #include #include "Types.h" -#include "knowhere/index/vector_index/VecIndex.h" #include "common/CDataType.h" #include "knowhere/index/Index.h" diff --git a/internal/core/src/common/Types.cpp b/internal/core/src/common/Types.cpp index 02db2032b3..6c0b5abac6 100644 --- a/internal/core/src/common/Types.cpp +++ b/internal/core/src/common/Types.cpp @@ -27,17 +27,17 @@ namespace milvus { using boost::algorithm::to_upper_copy; -namespace Metric = knowhere::Metric; +namespace metric = knowhere::metric; static const auto metric_bimap = [] { boost::bimap mapping; using pos = boost::bimap::value_type; - mapping.insert(pos(std::string(Metric::L2), MetricType::METRIC_L2)); - mapping.insert(pos(std::string(Metric::IP), MetricType::METRIC_INNER_PRODUCT)); - mapping.insert(pos(std::string(Metric::JACCARD), MetricType::METRIC_Jaccard)); - mapping.insert(pos(std::string(Metric::TANIMOTO), MetricType::METRIC_Tanimoto)); - mapping.insert(pos(std::string(Metric::HAMMING), MetricType::METRIC_Hamming)); - mapping.insert(pos(std::string(Metric::SUBSTRUCTURE), MetricType::METRIC_Substructure)); - mapping.insert(pos(std::string(Metric::SUPERSTRUCTURE), MetricType::METRIC_Superstructure)); + mapping.insert(pos(std::string(metric::L2), MetricType::METRIC_L2)); + mapping.insert(pos(std::string(metric::IP), MetricType::METRIC_INNER_PRODUCT)); + mapping.insert(pos(std::string(metric::JACCARD), MetricType::METRIC_Jaccard)); + mapping.insert(pos(std::string(metric::TANIMOTO), MetricType::METRIC_Tanimoto)); + mapping.insert(pos(std::string(metric::HAMMING), MetricType::METRIC_Hamming)); + mapping.insert(pos(std::string(metric::SUBSTRUCTURE), MetricType::METRIC_Substructure)); + mapping.insert(pos(std::string(metric::SUPERSTRUCTURE), MetricType::METRIC_Superstructure)); return mapping; }(); diff --git a/internal/core/src/index/BoolIndex.h b/internal/core/src/index/BoolIndex.h index c8d0713a58..20cfb90e4c 100644 --- a/internal/core/src/index/BoolIndex.h +++ b/internal/core/src/index/BoolIndex.h @@ -22,8 +22,8 @@ class BoolIndex : public ScalarIndexSort { public: void BuildWithDataset(const DatasetPtr& dataset) override { - auto size = dataset->Get(knowhere::meta::ROWS); - auto data = dataset->Get(knowhere::meta::TENSOR); + auto size = knowhere::GetDatasetRows(dataset); + auto data = knowhere::GetDatasetTensor(dataset); proto::schema::BoolArray arr; arr.ParseFromArray(data, size); Build(arr.data().size(), arr.data().data()); diff --git a/internal/core/src/index/ScalarIndex-inl.h b/internal/core/src/index/ScalarIndex-inl.h index f4f304c0b4..f949da1faa 100644 --- a/internal/core/src/index/ScalarIndex-inl.h +++ b/internal/core/src/index/ScalarIndex-inl.h @@ -9,9 +9,11 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License -#include #include +#include + #include "index/Meta.h" +#include "knowhere/index/vector_index/adapter/VectorAdapter.h" namespace milvus::scalar { template @@ -36,14 +38,14 @@ ScalarIndex::Query(const DatasetPtr& dataset) { } case OpType::In: { - auto n = dataset->Get(knowhere::meta::ROWS); - auto values = dataset->Get(knowhere::meta::TENSOR); + auto n = knowhere::GetDatasetRows(dataset); + auto values = knowhere::GetDatasetTensor(dataset); return In(n, reinterpret_cast(values)); } case OpType::NotIn: { - auto n = dataset->Get(knowhere::meta::ROWS); - auto values = dataset->Get(knowhere::meta::TENSOR); + auto n = knowhere::GetDatasetRows(dataset); + auto values = knowhere::GetDatasetTensor(dataset); return NotIn(n, reinterpret_cast(values)); } diff --git a/internal/core/src/index/ScalarIndexSort-inl.h b/internal/core/src/index/ScalarIndexSort-inl.h index b25659c079..b27ef7bc13 100644 --- a/internal/core/src/index/ScalarIndexSort-inl.h +++ b/internal/core/src/index/ScalarIndexSort-inl.h @@ -33,8 +33,8 @@ inline ScalarIndexSort::ScalarIndexSort(const size_t n, const T* values) : is template inline void ScalarIndexSort::BuildWithDataset(const DatasetPtr& dataset) { - auto size = dataset->Get(knowhere::meta::ROWS); - auto data = dataset->Get(knowhere::meta::TENSOR); + auto size = knowhere::GetDatasetRows(dataset); + auto data = knowhere::GetDatasetTensor(dataset); Build(size, reinterpret_cast(data)); } diff --git a/internal/core/src/index/StringIndex.h b/internal/core/src/index/StringIndex.h index f37fd6fffd..b8af2597c2 100644 --- a/internal/core/src/index/StringIndex.h +++ b/internal/core/src/index/StringIndex.h @@ -24,8 +24,8 @@ class StringIndex : public ScalarIndex { public: void BuildWithDataset(const DatasetPtr& dataset) override { - auto size = dataset->Get(knowhere::meta::ROWS); - auto data = dataset->Get(knowhere::meta::TENSOR); + auto size = knowhere::GetDatasetRows(dataset); + auto data = knowhere::GetDatasetTensor(dataset); proto::schema::StringArray arr; arr.ParseFromArray(data, size); diff --git a/internal/core/src/index/StringIndexMarisa.cpp b/internal/core/src/index/StringIndexMarisa.cpp index 416146649c..5b38ea560c 100644 --- a/internal/core/src/index/StringIndexMarisa.cpp +++ b/internal/core/src/index/StringIndexMarisa.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include "index/StringIndexMarisa.h" #include "index/Utils.h" diff --git a/internal/core/src/index/StringIndexSort.h b/internal/core/src/index/StringIndexSort.h index 5641cc804c..ac9592cc62 100644 --- a/internal/core/src/index/StringIndexSort.h +++ b/internal/core/src/index/StringIndexSort.h @@ -13,9 +13,10 @@ #include #include +#include "common/Utils.h" #include "index/ScalarIndexSort.h" #include "index/StringIndex.h" -#include "common/Utils.h" +#include "knowhere/index/vector_index/adapter/VectorAdapter.h" namespace milvus::scalar { // TODO: should inherit from StringIndex? @@ -23,8 +24,8 @@ class StringIndexSort : public ScalarIndexSort { public: void BuildWithDataset(const DatasetPtr& dataset) override { - auto size = dataset->Get(knowhere::meta::ROWS); - auto data = dataset->Get(knowhere::meta::TENSOR); + auto size = knowhere::GetDatasetRows(dataset); + auto data = knowhere::GetDatasetTensor(dataset); proto::schema::StringArray arr; arr.ParseFromArray(data, size); diff --git a/internal/core/src/indexbuilder/VecIndexCreator.cpp b/internal/core/src/indexbuilder/VecIndexCreator.cpp index 7609d8efec..ccb55b53a4 100644 --- a/internal/core/src/indexbuilder/VecIndexCreator.cpp +++ b/internal/core/src/indexbuilder/VecIndexCreator.cpp @@ -14,14 +14,15 @@ #include #include "exceptions/EasyAssert.h" -#include "pb/index_cgo_msg.pb.h" #include "indexbuilder/VecIndexCreator.h" #include "indexbuilder/utils.h" #include "knowhere/common/Timer.h" -#include "knowhere/common/Utils.h" +#include "knowhere/index/VecIndex.h" +#include "knowhere/index/VecIndexFactory.h" #include "knowhere/index/vector_index/ConfAdapterMgr.h" -#include "knowhere/index/vector_index/VecIndexFactory.h" +#include "knowhere/index/vector_index/adapter/VectorAdapter.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" +#include "pb/index_cgo_msg.pb.h" namespace milvus::indexbuilder { @@ -57,54 +58,56 @@ VecIndexCreator::parse_impl(const std::string& serialized_params_str, knowhere:: conf[key] = value; } - auto stoi_closure = [](const std::string& s) -> int { return std::stoi(s); }; - auto stof_closure = [](const std::string& s) -> float { return std::stof(s); }; + auto stoi_closure = [](const std::string_view& s) -> int { return std::stoi(std::string(s)); }; + auto stof_closure = [](const std::string_view& s) -> float { return std::stof(std::string(s)); }; /***************************** meta *******************************/ + check_parameter(conf, knowhere::meta::SLICE_SIZE, stoi_closure, std::optional{4}); check_parameter(conf, knowhere::meta::DIM, stoi_closure, std::nullopt); check_parameter(conf, knowhere::meta::TOPK, stoi_closure, std::nullopt); /***************************** IVF Params *******************************/ - check_parameter(conf, knowhere::IndexParams::nprobe, stoi_closure, std::nullopt); - check_parameter(conf, knowhere::IndexParams::nlist, stoi_closure, std::nullopt); - check_parameter(conf, knowhere::IndexParams::m, stoi_closure, std::nullopt); - check_parameter(conf, knowhere::IndexParams::nbits, stoi_closure, std::nullopt); - - /************************** NSG Parameter **************************/ - check_parameter(conf, knowhere::IndexParams::knng, stoi_closure, std::nullopt); - check_parameter(conf, knowhere::IndexParams::search_length, stoi_closure, std::nullopt); - check_parameter(conf, knowhere::IndexParams::out_degree, stoi_closure, std::nullopt); - check_parameter(conf, knowhere::IndexParams::candidate, stoi_closure, std::nullopt); - - /************************** HNSW Params *****************************/ - check_parameter(conf, knowhere::IndexParams::efConstruction, stoi_closure, std::nullopt); - check_parameter(conf, knowhere::IndexParams::M, stoi_closure, std::nullopt); - check_parameter(conf, knowhere::IndexParams::ef, stoi_closure, std::nullopt); - - /************************** Annoy Params *****************************/ - check_parameter(conf, knowhere::IndexParams::n_trees, stoi_closure, std::nullopt); - check_parameter(conf, knowhere::IndexParams::search_k, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::NPROBE, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::NLIST, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::M, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::NBITS, stoi_closure, std::nullopt); /************************** PQ Params *****************************/ - check_parameter(conf, knowhere::IndexParams::PQM, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::PQ_M, stoi_closure, std::nullopt); +#ifdef MILVUS_SUPPORT_NSG + /************************** NSG Parameter **************************/ + check_parameter(conf, knowhere::indexparam::KNNG, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::SEARCH_LENGTH, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::OUT_DEGREE, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::CANDIDATE, stoi_closure, std::nullopt); +#endif + + /************************** HNSW Params *****************************/ + check_parameter(conf, knowhere::indexparam::EFCONSTRUCTION, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::HNSW_M, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::EF, stoi_closure, std::nullopt); + + /************************** Annoy Params *****************************/ + check_parameter(conf, knowhere::indexparam::N_TREES, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::SEARCH_K, stoi_closure, std::nullopt); + +#ifdef MILVUS_SUPPORT_NGT /************************** NGT Params *****************************/ - check_parameter(conf, knowhere::IndexParams::edge_size, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::EDGE_SIZE, stoi_closure, std::nullopt); /************************** NGT Search Params *****************************/ - check_parameter(conf, knowhere::IndexParams::epsilon, stof_closure, std::nullopt); - check_parameter(conf, knowhere::IndexParams::max_search_edges, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::EPSILON, stof_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::MAX_SEARCH_EDGES, stoi_closure, std::nullopt); /************************** NGT_PANNG Params *****************************/ - check_parameter(conf, knowhere::IndexParams::forcedly_pruned_edge_size, stoi_closure, std::nullopt); - check_parameter(conf, knowhere::IndexParams::selectively_pruned_edge_size, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::FORCEDLY_PRUNED_EDGE_SIZE, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::SELECTIVELY_PRUNED_EDGE_SIZE, stoi_closure, std::nullopt); /************************** NGT_ONNG Params *****************************/ - check_parameter(conf, knowhere::IndexParams::outgoing_edge_size, stoi_closure, std::nullopt); - check_parameter(conf, knowhere::IndexParams::incoming_edge_size, stoi_closure, std::nullopt); - - /************************** Serialize Params *******************************/ - check_parameter(conf, knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, stoi_closure, std::optional{4}); + check_parameter(conf, knowhere::indexparam::OUTGOING_EDGE_SIZE, stoi_closure, std::nullopt); + check_parameter(conf, knowhere::indexparam::INCOMING_EDGE_SIZE, stoi_closure, std::nullopt); +#endif } void @@ -121,24 +124,24 @@ VecIndexCreator::parse() { template void VecIndexCreator::check_parameter(knowhere::Config& conf, - const std::string& key, + const std::string_view& key, std::function fn, std::optional default_v) { if (!conf.contains(key)) { if (default_v.has_value()) { - conf[key] = default_v.value(); + conf[std::string(key)] = default_v.value(); } } else { - auto value = conf[key]; - conf[key] = fn(value); + auto value = conf[std::string(key)]; + conf[std::string(key)] = fn(value); } } template std::optional -VecIndexCreator::get_config_by_name(std::string name) { +VecIndexCreator::get_config_by_name(std::string_view name) { if (config_.contains(name)) { - return {config_[name].get()}; + return knowhere::GetValueFromConfig(config_, name); } return std::nullopt; } @@ -154,10 +157,10 @@ void VecIndexCreator::BuildWithoutIds(const knowhere::DatasetPtr& dataset) { auto index_type = get_index_type(); auto index_mode = get_index_mode(); - config_[knowhere::meta::ROWS] = dataset->Get(knowhere::meta::ROWS); + knowhere::SetMetaRows(config_, knowhere::GetDatasetRows(dataset)); if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) { - if (!config_.contains(knowhere::IndexParams::nbits)) { - config_[knowhere::IndexParams::nbits] = 8; + if (!config_.contains(knowhere::indexparam::NBITS)) { + knowhere::SetIndexParamNbits(config_, 8); } } auto conf_adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type); @@ -187,14 +190,12 @@ VecIndexCreator::BuildWithoutIds(const knowhere::DatasetPtr& dataset) { void VecIndexCreator::BuildWithIds(const knowhere::DatasetPtr& dataset) { - AssertInfo(dataset->data().find(knowhere::meta::IDS) != dataset->data().end(), - "[VecIndexCreator]Can't find ids field in dataset"); auto index_type = get_index_type(); auto index_mode = get_index_mode(); - config_[knowhere::meta::ROWS] = dataset->Get(knowhere::meta::ROWS); + knowhere::SetMetaRows(config_, knowhere::GetDatasetRows(dataset)); if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) { - if (!config_.contains(knowhere::IndexParams::nbits)) { - config_[knowhere::IndexParams::nbits] = 8; + if (!config_.contains(knowhere::indexparam::NBITS)) { + knowhere::SetIndexParamNbits(config_, 8); } } auto conf_adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type); @@ -212,9 +213,9 @@ void VecIndexCreator::StoreRawData(const knowhere::DatasetPtr& dataset) { auto index_type = get_index_type(); if (is_in_nm_list(index_type)) { - auto tensor = dataset->Get(knowhere::meta::TENSOR); - auto row_num = dataset->Get(knowhere::meta::ROWS); - auto dim = dataset->Get(knowhere::meta::DIM); + auto tensor = knowhere::GetDatasetTensor(dataset); + auto row_num = knowhere::GetDatasetRows(dataset); + auto dim = knowhere::GetDatasetDim(dataset); int64_t data_size; if (is_in_bin_list(index_type)) { data_size = dim / 8 * row_num; @@ -263,20 +264,20 @@ VecIndexCreator::get_index_type() { // knowhere bug here // the index_type of all ivf-based index will change to ivf flat after loaded auto type = get_config_by_name("index_type"); - return type.has_value() ? type.value() : knowhere::IndexEnum::INDEX_FAISS_IVFPQ; + return type.has_value() ? type.value() : std::string(knowhere::IndexEnum::INDEX_FAISS_IVFPQ); } std::string VecIndexCreator::get_metric_type() { - auto type = get_config_by_name(knowhere::Metric::TYPE); + auto type = get_config_by_name(knowhere::meta::METRIC_TYPE); if (type.has_value()) { return type.value(); } else { auto index_type = get_index_type(); if (is_in_bin_list(index_type)) { - return knowhere::Metric::JACCARD; + return std::string(knowhere::metric::JACCARD); } else { - return knowhere::Metric::L2; + return std::string(knowhere::metric::L2); } } } @@ -293,8 +294,8 @@ VecIndexCreator::get_index_mode() { int64_t VecIndexCreator::get_index_file_slice_size() { - if (config_.contains(knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE)) { - return config_[knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE].get(); + if (knowhere::CheckKeyInConfig(config_, knowhere::meta::SLICE_SIZE)) { + return knowhere::GetMetaSliceSize(config_); } return knowhere::index_file_slice_size; // by default } @@ -322,10 +323,10 @@ VecIndexCreator::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere:: } auto res = index_->Query(dataset, conf, nullptr); - auto ids = res->Get(knowhere::meta::IDS); - auto distances = res->Get(knowhere::meta::DISTANCE); - auto nq = dataset->Get(knowhere::meta::ROWS); - auto k = config_[knowhere::meta::TOPK].get(); + auto ids = knowhere::GetDatasetIDs(res); + auto distances = knowhere::GetDatasetDistance(res); + auto nq = knowhere::GetDatasetRows(dataset); + auto k = knowhere::GetMetaTopk(config_); auto query_res = std::make_unique(); query_res->nq = nq; diff --git a/internal/core/src/indexbuilder/VecIndexCreator.h b/internal/core/src/indexbuilder/VecIndexCreator.h index 0e1e9b5eb8..07d21fbc4d 100644 --- a/internal/core/src/indexbuilder/VecIndexCreator.h +++ b/internal/core/src/indexbuilder/VecIndexCreator.h @@ -16,9 +16,9 @@ #include #include -#include "knowhere/index/vector_index/VecIndex.h" -#include "knowhere/common/BinarySet.h" #include "indexbuilder/IndexCreatorBase.h" +#include "knowhere/common/BinarySet.h" +#include "knowhere/index/VecIndex.h" namespace milvus::indexbuilder { @@ -74,7 +74,7 @@ class VecIndexCreator : public IndexCreatorBase { template std::optional - get_config_by_name(std::string name); + get_config_by_name(std::string_view name); void StoreRawData(const knowhere::DatasetPtr& dataset); @@ -85,7 +85,7 @@ class VecIndexCreator : public IndexCreatorBase { template void check_parameter(knowhere::Config& conf, - const std::string& key, + const std::string_view& key, std::function fn, std::optional default_v = std::nullopt); diff --git a/internal/core/src/indexbuilder/utils.cpp b/internal/core/src/indexbuilder/utils.cpp index d77efdb71b..9a92ce7f41 100644 --- a/internal/core/src/indexbuilder/utils.cpp +++ b/internal/core/src/indexbuilder/utils.cpp @@ -15,15 +15,12 @@ #include #include #include -#include - -#include "knowhere/index/IndexType.h" namespace milvus::indexbuilder { -std::vector +std::vector NM_List() { - static std::vector ret{ + static std::vector ret{ knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, #ifdef MILVUS_SUPPORT_NSG knowhere::IndexEnum::INDEX_NSG, @@ -33,18 +30,18 @@ NM_List() { return ret; } -std::vector +std::vector BIN_List() { - static std::vector ret{ + static std::vector ret{ knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, }; return ret; } -std::vector +std::vector Need_ID_List() { - static std::vector ret{ + static std::vector ret{ // knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, // knowhere::IndexEnum::INDEX_NSG, }; @@ -52,9 +49,9 @@ Need_ID_List() { return ret; } -std::vector +std::vector Need_BuildAll_list() { - static std::vector ret{ + static std::vector ret{ #ifdef MILVUS_SUPPORT_NSG knowhere::IndexEnum::INDEX_NSG, #endif @@ -62,10 +59,10 @@ Need_BuildAll_list() { return ret; } -std::vector> +std::vector> unsupported_index_combinations() { - static std::vector> ret{ - std::make_tuple(std::string(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT), std::string(knowhere::Metric::L2)), + static std::vector> ret{ + std::make_tuple(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::metric::L2), }; return ret; } @@ -79,28 +76,28 @@ is_in_list(const T& t, std::function()> list_func) { bool is_in_bin_list(const knowhere::IndexType& index_type) { - return is_in_list(index_type, BIN_List); + return is_in_list(index_type, BIN_List); } bool is_in_nm_list(const knowhere::IndexType& index_type) { - return is_in_list(index_type, NM_List); + return is_in_list(index_type, NM_List); } bool is_in_need_build_all_list(const knowhere::IndexType& index_type) { - return is_in_list(index_type, Need_BuildAll_list); + return is_in_list(index_type, Need_BuildAll_list); } bool is_in_need_id_list(const knowhere::IndexType& index_type) { - return is_in_list(index_type, Need_ID_List); + return is_in_list(index_type, Need_ID_List); } bool is_unsupported(const knowhere::IndexType& index_type, const knowhere::MetricType& metric_type) { - return is_in_list>(std::make_tuple(index_type, metric_type), - unsupported_index_combinations); + return is_in_list>(std::make_tuple(index_type, metric_type), + unsupported_index_combinations); } } // namespace milvus::indexbuilder diff --git a/internal/core/src/indexbuilder/utils.h b/internal/core/src/indexbuilder/utils.h index 60f7c3879d..2f7ea5327b 100644 --- a/internal/core/src/indexbuilder/utils.h +++ b/internal/core/src/indexbuilder/utils.h @@ -16,25 +16,26 @@ #include #include #include -#include +#include #include "knowhere/index/IndexType.h" +#include "knowhere/index/vector_index/helpers/IndexParameter.h" namespace milvus::indexbuilder { -std::vector +std::vector NM_List(); -std::vector +std::vector BIN_List(); -std::vector +std::vector Need_ID_List(); -std::vector +std::vector Need_BuildAll_list(); -std::vector> +std::vector> unsupported_index_combinations(); template diff --git a/internal/core/src/query/PlanNode.h b/internal/core/src/query/PlanNode.h index 085f419b36..61ce2a454f 100644 --- a/internal/core/src/query/PlanNode.h +++ b/internal/core/src/query/PlanNode.h @@ -18,7 +18,7 @@ #include #include "Expr.h" -#include "utils/Json.h" +#include "knowhere/common/Config.h" namespace milvus::query { @@ -39,7 +39,7 @@ struct SearchInfo { int64_t round_decimal_; FieldId field_id_; MetricType metric_type_; - nlohmann::json search_params_; + knowhere::Config search_params_; }; struct VectorPlanNode : PlanNode { diff --git a/internal/core/src/query/SearchOnIndex.cpp b/internal/core/src/query/SearchOnIndex.cpp index 8ccc27951d..f13edcc58a 100644 --- a/internal/core/src/query/SearchOnIndex.cpp +++ b/internal/core/src/query/SearchOnIndex.cpp @@ -30,8 +30,8 @@ SearchOnIndex(const dataset::SearchDataset& search_dataset, auto& indexing_nonconst = const_cast(indexing); auto ans = indexing_nonconst.Query(dataset, search_conf, bitset); - auto dis = ans->Get(knowhere::meta::DISTANCE); - auto uids = ans->Get(knowhere::meta::IDS); + auto dis = knowhere::GetDatasetDistance(ans); + auto uids = knowhere::GetDatasetIDs(ans); SubSearchResult sub_qr(num_queries, topK, metric_type, round_decimal); std::copy_n(dis, num_queries * topK, sub_qr.get_distances()); diff --git a/internal/core/src/query/SearchOnIndex.h b/internal/core/src/query/SearchOnIndex.h index dd3152fd26..541bcfb161 100644 --- a/internal/core/src/query/SearchOnIndex.h +++ b/internal/core/src/query/SearchOnIndex.h @@ -12,7 +12,7 @@ #pragma once #include "common/BitsetView.h" -#include "knowhere/index/vector_index/VecIndex.h" +#include "knowhere/index/VecIndex.h" #include "query/SubSearchResult.h" #include "query/helper.h" diff --git a/internal/core/src/query/SearchOnSealed.cpp b/internal/core/src/query/SearchOnSealed.cpp index 87c3c566f7..a14ac4df44 100644 --- a/internal/core/src/query/SearchOnSealed.cpp +++ b/internal/core/src/query/SearchOnSealed.cpp @@ -11,7 +11,7 @@ #include -#include "knowhere/index/vector_index/VecIndex.h" +#include "knowhere/index/VecIndex.h" #include "knowhere/index/vector_index/ConfAdapter.h" #include "knowhere/index/vector_index/ConfAdapterMgr.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" @@ -46,17 +46,20 @@ SearchOnSealed(const Schema& schema, auto ds = knowhere::GenDataset(num_queries, dim, query_data); auto conf = search_info.search_params_; - conf[knowhere::meta::TOPK] = search_info.topk_; - conf[knowhere::Metric::TYPE] = MetricTypeToName(field_indexing->metric_type_); + knowhere::SetMetaTopk(conf, search_info.topk_); + knowhere::SetMetaMetricType(conf, MetricTypeToName(field_indexing->metric_type_)); auto index_type = field_indexing->indexing_->index_type(); auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type); - AssertInfo(adapter->CheckSearch(conf, index_type, field_indexing->indexing_->index_mode()), - "[SearchOnSealed]Search params check failed"); + try { + adapter->CheckSearch(conf, index_type, field_indexing->indexing_->index_mode()); + } catch (std::exception& e) { + AssertInfo(false, e.what()); + } return field_indexing->indexing_->Query(ds, conf, bitset); }(); - auto ids = final->Get(knowhere::meta::IDS); - auto distances = final->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(final); + float* distances = (float*)knowhere::GetDatasetDistance(final); auto total_num = num_queries * topk; diff --git a/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp b/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp index d913ed2d53..e37d56fe60 100644 --- a/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp +++ b/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp @@ -38,14 +38,14 @@ InferIndexType(const Json& search_params) { // ngtpanng / ngtonng -> max_search_edges / epsilon static const std::map key_list = [] { std::map list; - namespace ip = knowhere::IndexParams; + namespace ip = knowhere::indexparam; namespace ie = knowhere::IndexEnum; - list.emplace(ip::nprobe, ie::INDEX_FAISS_IVFFLAT); + list.emplace(ip::NPROBE, ie::INDEX_FAISS_IVFFLAT); #ifdef MILVUS_SUPPORT_NSG list.emplace(ip::search_length, ie::INDEX_NSG); #endif - list.emplace(ip::ef, ie::INDEX_HNSW); - list.emplace(ip::search_k, ie::INDEX_ANNOY); + list.emplace(ip::EF, ie::INDEX_HNSW); + list.emplace(ip::SEARCH_K, ie::INDEX_ANNOY); #ifdef MILVUS_SUPPORT_NGT list.emplace(ip::max_search_edges, ie::INDEX_NGTONNG); list.emplace(ip::epsilon, ie::INDEX_NGTONNG); @@ -64,9 +64,9 @@ InferIndexType(const Json& search_params) { static knowhere::IndexType InferBinaryIndexType(const Json& search_params) { - namespace ip = knowhere::IndexParams; + namespace ip = knowhere::indexparam; namespace ie = knowhere::IndexEnum; - if (search_params.contains(ip::nprobe)) { + if (search_params.contains(ip::NPROBE)) { return ie::INDEX_FAISS_BIN_IVFFLAT; } else { return ie::INDEX_FAISS_BIN_IDMAP; @@ -82,7 +82,7 @@ VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) { // mock the api, topk will be passed from placeholder auto params_copy = search_params; - params_copy[knowhere::meta::TOPK] = 10; + knowhere::SetMetaTopk(params_copy, 10); // NOTE: the second parameter is not checked in knowhere, may be redundant auto passed = adapter->CheckSearch(params_copy, inferred_type, index_mode); @@ -100,7 +100,7 @@ VerifyPlanNodeVisitor::visit(BinaryVectorANNS& node) { // mock the api, topk will be passed from placeholder auto params_copy = search_params; - params_copy[knowhere::meta::TOPK] = 10; + knowhere::SetMetaTopk(params_copy, 10); // NOTE: the second parameter is not checked in knowhere, may be redundant auto passed = adapter->CheckSearch(params_copy, inferred_type, index_mode); diff --git a/internal/core/src/segcore/FieldIndexing.cpp b/internal/core/src/segcore/FieldIndexing.cpp index c428692562..02e7df607b 100644 --- a/internal/core/src/segcore/FieldIndexing.cpp +++ b/internal/core/src/segcore/FieldIndexing.cpp @@ -54,8 +54,8 @@ VectorFieldIndexing::get_build_params() const { auto base_params = config.build_params; AssertInfo(base_params.count("nlist"), "Can't get nlist from index params"); - base_params[knowhere::meta::DIM] = field_meta_.get_dim(); - base_params[knowhere::Metric::TYPE] = type_name; + knowhere::SetMetaDim(base_params, field_meta_.get_dim()); + knowhere::SetMetaMetricType(base_params, type_name); return base_params; } @@ -71,8 +71,8 @@ VectorFieldIndexing::get_search_params(int top_K) const { auto base_params = config.search_params; AssertInfo(base_params.count("nprobe"), "Can't get nprobe from base params"); - base_params[knowhere::meta::TOPK] = top_K; - base_params[knowhere::Metric::TYPE] = type_name; + knowhere::SetMetaTopk(base_params, top_K); + knowhere::SetMetaMetricType(base_params, type_name); return base_params; } diff --git a/internal/core/src/segcore/FieldIndexing.h b/internal/core/src/segcore/FieldIndexing.h index 33af112c11..6fa13a70ea 100644 --- a/internal/core/src/segcore/FieldIndexing.h +++ b/internal/core/src/segcore/FieldIndexing.h @@ -22,7 +22,7 @@ #include "AckResponder.h" #include "InsertRecord.h" #include "common/Schema.h" -#include "knowhere/index/vector_index/VecIndex.h" +#include "knowhere/index/VecIndex.h" #include "segcore/SegcoreConfig.h" namespace milvus::segcore { diff --git a/internal/core/src/segcore/SealedIndexingRecord.h b/internal/core/src/segcore/SealedIndexingRecord.h index bf0285a845..162433aee3 100644 --- a/internal/core/src/segcore/SealedIndexingRecord.h +++ b/internal/core/src/segcore/SealedIndexingRecord.h @@ -20,7 +20,7 @@ #include "common/Types.h" #include "exceptions/EasyAssert.h" -#include "knowhere/index/vector_index/VecIndex.h" +#include "knowhere/index/VecIndex.h" namespace milvus::segcore { diff --git a/internal/core/src/segcore/SegmentGrowingImpl.h b/internal/core/src/segcore/SegmentGrowingImpl.h index 4b44bc2469..83d3486ce9 100644 --- a/internal/core/src/segcore/SegmentGrowingImpl.h +++ b/internal/core/src/segcore/SegmentGrowingImpl.h @@ -30,7 +30,6 @@ #include "SegmentGrowing.h" #include "exceptions/EasyAssert.h" -#include "knowhere/index/vector_index/VecIndex.h" #include "query/PlanNode.h" #include "query/deprecated/GeneralQuery.h" #include "utils/Status.h" diff --git a/internal/core/src/segcore/SegmentInterface.h b/internal/core/src/segcore/SegmentInterface.h index 6eef3b9c27..f17da8560d 100644 --- a/internal/core/src/segcore/SegmentInterface.h +++ b/internal/core/src/segcore/SegmentInterface.h @@ -27,7 +27,6 @@ #include "common/LoadInfo.h" #include "common/BitsetView.h" #include "common/QueryResult.h" -#include "knowhere/index/vector_index/VecIndex.h" #include "query/Plan.h" #include "query/PlanNode.h" #include "pb/schema.pb.h" diff --git a/internal/core/src/segcore/load_index_c.cpp b/internal/core/src/segcore/load_index_c.cpp index 365ac4f6d2..7270325456 100644 --- a/internal/core/src/segcore/load_index_c.cpp +++ b/internal/core/src/segcore/load_index_c.cpp @@ -9,13 +9,13 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License +#include "common/CDataType.h" #include "common/LoadInfo.h" #include "exceptions/EasyAssert.h" -#include "knowhere/common/BinarySet.h" -#include "knowhere/index/vector_index/VecIndexFactory.h" -#include "segcore/load_index_c.h" #include "index/IndexFactory.h" -#include "common/CDataType.h" +#include "knowhere/common/BinarySet.h" +#include "knowhere/index/VecIndexFactory.h" +#include "segcore/load_index_c.h" CStatus NewLoadIndexInfo(CLoadIndexInfo* c_load_index_info) { diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt index d6b4847170..ef80e1fd84 100644 --- a/internal/core/thirdparty/knowhere/CMakeLists.txt +++ b/internal/core/thirdparty/knowhere/CMakeLists.txt @@ -11,8 +11,8 @@ # or implied. See the License for the specific language governing permissions and limitations under the License. #------------------------------------------------------------------------------- -set( KNOWHERE_VERSION v1.1.5 ) -set( KNOWHERE_SOURCE_MD5 "5baa7ecaea0e9e5c9f3f440cf8757d3c") +set( KNOWHERE_VERSION v1.1.12 ) +set( KNOWHERE_SOURCE_MD5 "816dadc4d41fc9a4eab1fb9c9a7c2736") if ( DEFINED ENV{MILVUS_KNOWHERE_URL} ) set( KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}" ) diff --git a/internal/core/unittest/bench/bench_indexbuilder.cpp b/internal/core/unittest/bench/bench_indexbuilder.cpp index a450e91bdf..0dea9915e3 100644 --- a/internal/core/unittest/bench/bench_indexbuilder.cpp +++ b/internal/core/unittest/bench/bench_indexbuilder.cpp @@ -36,7 +36,7 @@ auto index_type_collections = [] { auto metric_type_collections = [] { static std::map collections{ - {0, knowhere::Metric::L2}, + {0, knowhere::metric::L2}, }; return collections; }(); diff --git a/internal/core/unittest/test_bool_index.cpp b/internal/core/unittest/test_bool_index.cpp index 1f91ab6061..841d8bd704 100644 --- a/internal/core/unittest/test_bool_index.cpp +++ b/internal/core/unittest/test_bool_index.cpp @@ -33,9 +33,9 @@ class BoolIndexTest : public ::testing::Test { void TearDown() override { - delete[](char*)(all_true_ds->Get(knowhere::meta::TENSOR)); - delete[](char*) all_false_ds->Get(knowhere::meta::TENSOR); - delete[](char*) half_ds->Get(knowhere::meta::TENSOR); + delete[](char*)(knowhere::GetDatasetTensor(all_true_ds)); + delete[](char*)(knowhere::GetDatasetTensor(all_false_ds)); + delete[](char*)(knowhere::GetDatasetTensor(half_ds)); } protected: diff --git a/internal/core/unittest/test_c_api.cpp b/internal/core/unittest/test_c_api.cpp index 3ff1550f08..3ba8c49c91 100644 --- a/internal/core/unittest/test_c_api.cpp +++ b/internal/core/unittest/test_c_api.cpp @@ -9,21 +9,21 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License -#include +#include #include #include +#include #include #include #include #include -#include -#include -#include -#include -#include #include "common/LoadInfo.h" #include "common/memory_c.h" +#include "knowhere/index/VecIndexFactory.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" +#include "knowhere/index/vector_index/helpers/IndexParameter.h" +#include "knowhere/index/vector_index/adapter/VectorAdapter.h" #include "pb/plan.pb.h" #include "query/ExprImpl.h" #include "segcore/Collection.h" @@ -148,7 +148,7 @@ generate_collection_schema(std::string metric_type, int dim, bool is_binary) { } VecIndexPtr -generate_index(void* raw_data, knowhere::Config conf, int64_t dim, int64_t topK, int64_t N, std::string index_type) { +generate_index(void* raw_data, knowhere::Config conf, int64_t dim, int64_t topK, int64_t N, knowhere::IndexType index_type) { auto indexing = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type, knowhere::IndexMode::MODE_CPU); auto database = knowhere::GenDataset(N, dim, raw_data); @@ -1241,14 +1241,16 @@ TEST(CApiTest, LoadIndexInfo) { auto N = 1024 * 10; auto [raw_data, timestamps, uids] = generate_data(N); auto indexing = std::make_shared(); - auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, - {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 4}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 4}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, + {knowhere::meta::DEVICE_ID, 0} + }; auto database = knowhere::GenDataset(N, DIM, raw_data.data()); indexing->Train(database, conf); @@ -1284,14 +1286,16 @@ TEST(CApiTest, LoadIndex_Search) { auto num_query = 100; auto [raw_data, timestamps, uids] = generate_data(N); auto indexing = std::make_shared(); - auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, - {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 4}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 4}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, + {knowhere::meta::DEVICE_ID, 0} + }; auto database = knowhere::GenDataset(N, DIM, raw_data.data()); indexing->Train(database, conf); @@ -1317,8 +1321,8 @@ TEST(CApiTest, LoadIndex_Search) { auto result = indexing->Query(query_dataset, conf, nullptr); - auto ids = result->Get(knowhere::meta::IDS); - auto dis = result->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result); + auto dis = knowhere::GetDatasetDistance(result); // for (int i = 0; i < std::min(num_query * K, 100); ++i) { // std::cout << ids[i] << "->" << dis[i] << std::endl; //} @@ -1386,21 +1390,23 @@ TEST(CApiTest, Indexing_Without_Predicate) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, - {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, + {knowhere::meta::DEVICE_ID, 0} + }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(knowhere::meta::IDS); - auto dis = result_on_index->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result_on_index); + auto dis = knowhere::GetDatasetDistance(result_on_index); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -1511,21 +1517,23 @@ TEST(CApiTest, Indexing_Expr_Without_Predicate) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, - {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, + {knowhere::meta::DEVICE_ID, 0} + }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(knowhere::meta::IDS); - auto dis = result_on_index->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result_on_index); + auto dis = knowhere::GetDatasetDistance(result_on_index); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -1653,22 +1661,23 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, - {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, + {knowhere::meta::DEVICE_ID, 0}}; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(knowhere::meta::IDS); - auto dis = result_on_index->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result_on_index); + auto dis = knowhere::GetDatasetDistance(result_on_index); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -1810,22 +1819,24 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, - {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, + {knowhere::meta::DEVICE_ID, 0} + }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(knowhere::meta::IDS); - auto dis = result_on_index->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result_on_index); + auto dis = knowhere::GetDatasetDistance(result_on_index); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -1951,22 +1962,24 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, - {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, + {knowhere::meta::DEVICE_ID, 0} + }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(knowhere::meta::IDS); - auto dis = result_on_index->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result_on_index); + auto dis = knowhere::GetDatasetDistance(result_on_index); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2101,22 +2114,24 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Term) { assert(res_before_load_index.error_code == Success); // load index to segment - auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, - {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, + {knowhere::meta::DEVICE_ID, 0} + }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(knowhere::meta::IDS); - auto dis = result_on_index->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result_on_index); + auto dis = knowhere::GetDatasetDistance(result_on_index); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2244,13 +2259,13 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) { // load index to segment auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::JACCARD}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::JACCARD}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT); @@ -2258,8 +2273,8 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) { // gen query dataset auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(knowhere::meta::IDS); - auto dis = result_on_index->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result_on_index); + auto dis = knowhere::GetDatasetDistance(result_on_index); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2400,13 +2415,13 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) { // load index to segment auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::JACCARD}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::JACCARD}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT); @@ -2414,8 +2429,8 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) { // gen query dataset auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(knowhere::meta::IDS); - auto dis = result_on_index->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result_on_index); + auto dis = knowhere::GetDatasetDistance(result_on_index); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2543,13 +2558,13 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) { // load index to segment auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::JACCARD}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::JACCARD}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT); @@ -2557,8 +2572,8 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) { // gen query dataset auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(knowhere::meta::IDS); - auto dis = result_on_index->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result_on_index); + auto dis = knowhere::GetDatasetDistance(result_on_index); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2708,13 +2723,13 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) { // load index to segment auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::JACCARD}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::JACCARD}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT); @@ -2722,8 +2737,8 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) { // gen query dataset auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(knowhere::meta::IDS); - auto dis = result_on_index->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result_on_index); + auto dis = knowhere::GetDatasetDistance(result_on_index); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2890,22 +2905,24 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) { Timestamp time = 10000000; // load index to segment - auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, - {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, + {knowhere::meta::DEVICE_ID, 0} + }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); // gen query dataset auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(knowhere::meta::IDS); - auto dis = result_on_index->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result_on_index); + auto dis = knowhere::GetDatasetDistance(result_on_index); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { @@ -2933,8 +2950,8 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) { auto query_dataset2 = knowhere::GenDataset(num_queries, DIM, query_ptr); auto index = std::dynamic_pointer_cast(load_index_info->index); auto result_on_index2 = index->Query(query_dataset2, conf, nullptr); - auto ids2 = result_on_index2->Get(knowhere::meta::IDS); - auto dis2 = result_on_index2->Get(knowhere::meta::DISTANCE); + auto ids2 = knowhere::GetDatasetIDs(result_on_index2); + auto dis2 = knowhere::GetDatasetDistance(result_on_index2); auto c_counter_field_data = CLoadFieldDataInfo{ 101, @@ -3185,14 +3202,16 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) { Timestamp time = 10000000; // load index to segment - auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, - {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, + {knowhere::meta::DEVICE_ID, 0} + }; auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ); @@ -3218,8 +3237,8 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) { auto query_dataset2 = knowhere::GenDataset(num_queries, DIM, query_ptr); auto index = std::dynamic_pointer_cast(load_index_info->index); auto result_on_index2 = index->Query(query_dataset2, conf, nullptr); - auto ids2 = result_on_index2->Get(knowhere::meta::IDS); - auto dis2 = result_on_index2->Get(knowhere::meta::DISTANCE); + auto ids2 = knowhere::GetDatasetIDs(result_on_index2); + auto dis2 = knowhere::GetDatasetDistance(result_on_index2); status = UpdateSealedSegmentIndex(segment, c_load_index_info); assert(status.error_code == Success); @@ -3254,8 +3273,8 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) { // gen query dataset auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr); auto result_on_index = indexing->Query(query_dataset, conf, nullptr); - auto ids = result_on_index->Get(knowhere::meta::IDS); - auto dis = result_on_index->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result_on_index); + auto dis = knowhere::GetDatasetDistance(result_on_index); std::vector vec_ids(ids, ids + TOPK * num_queries); std::vector vec_dis; for (int j = 0; j < TOPK * num_queries; ++j) { diff --git a/internal/core/unittest/test_index_c_api.cpp b/internal/core/unittest/test_index_c_api.cpp index 9da88bebce..6e63005f01 100644 --- a/internal/core/unittest/test_index_c_api.cpp +++ b/internal/core/unittest/test_index_c_api.cpp @@ -32,7 +32,7 @@ constexpr int NB = 10; TEST(FloatVecIndex, All) { auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFPQ; - auto metric_type = knowhere::Metric::L2; + auto metric_type = knowhere::metric::L2; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -83,7 +83,7 @@ TEST(FloatVecIndex, All) { TEST(BinaryVecIndex, All) { auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT; - auto metric_type = knowhere::Metric::JACCARD; + auto metric_type = knowhere::metric::JACCARD; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -159,8 +159,7 @@ TEST(CBoolIndexTest, All) { ASSERT_EQ(Success, status.error_code); } { - status = BuildScalarIndex(index, half_ds->Get(knowhere::meta::ROWS), - half_ds->Get(knowhere::meta::TENSOR)); + status = BuildScalarIndex(index, knowhere::GetDatasetRows(half_ds), knowhere::GetDatasetTensor(half_ds)); ASSERT_EQ(Success, status.error_code); } { @@ -185,7 +184,7 @@ TEST(CBoolIndexTest, All) { } } - delete[](char*) half_ds->Get(knowhere::meta::TENSOR); + delete[](char*) knowhere::GetDatasetTensor(half_ds); } // TODO: more scalar type. @@ -262,8 +261,7 @@ TEST(CStringIndexTest, All) { ASSERT_EQ(Success, status.error_code); } { - status = BuildScalarIndex(index, str_ds->Get(knowhere::meta::ROWS), - str_ds->Get(knowhere::meta::TENSOR)); + status = BuildScalarIndex(index, knowhere::GetDatasetRows(str_ds), knowhere::GetDatasetTensor(str_ds)); ASSERT_EQ(Success, status.error_code); } { @@ -288,6 +286,6 @@ TEST(CStringIndexTest, All) { } } - delete[](char*) str_ds->Get(knowhere::meta::TENSOR); + delete[](char*) knowhere::GetDatasetTensor(str_ds); } #endif diff --git a/internal/core/unittest/test_index_wrapper.cpp b/internal/core/unittest/test_index_wrapper.cpp index 159dc66de3..090b21b64b 100644 --- a/internal/core/unittest/test_index_wrapper.cpp +++ b/internal/core/unittest/test_index_wrapper.cpp @@ -41,7 +41,7 @@ class IndexWrapperTest : public ::testing::TestWithParam { metric_type = param.second; std::tie(type_params, index_params) = generate_params(index_type, metric_type); - std::map is_binary_map = { + std::map is_binary_map = { {knowhere::IndexEnum::INDEX_FAISS_IDMAP, false}, {knowhere::IndexEnum::INDEX_FAISS_IVFPQ, false}, {knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, false}, @@ -109,7 +109,7 @@ class IndexWrapperTest : public ::testing::TestWithParam { TEST(PQ, Build) { auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFPQ; - auto metric_type = knowhere::Metric::L2; + auto metric_type = knowhere::metric::L2; auto conf = generate_conf(index_type, metric_type); auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto dataset = GenDataset(NB, metric_type, false); @@ -121,7 +121,7 @@ TEST(PQ, Build) { TEST(IVFFLATNM, Build) { auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; - auto metric_type = knowhere::Metric::L2; + auto metric_type = knowhere::metric::L2; auto conf = generate_conf(index_type, metric_type); auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto dataset = GenDataset(NB, metric_type, false); @@ -135,7 +135,7 @@ TEST(IVFFLATNM, Query) { knowhere::KnowhereConfig::SetStatisticsLevel(3); auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; - auto metric_type = knowhere::Metric::L2; + auto metric_type = knowhere::metric::L2; auto conf = generate_conf(index_type, metric_type); auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto dataset = GenDataset(NB, metric_type, false); @@ -153,8 +153,7 @@ TEST(IVFFLATNM, Query) { auto xq_dataset = knowhere::GenDataset(NQ, DIM, xq_data.data()); auto result = index->Query(xq_dataset, conf, nullptr); - index->UpdateIndexSize(); - ASSERT_GT(index->IndexSize(), 0); + ASSERT_GT(index->Size(), 0); auto stats = index->GetStatistics(); ASSERT_TRUE(stats != nullptr); @@ -164,7 +163,7 @@ TEST(IVFFLATNM, Query) { #ifdef MILVUS_SUPPORT_NSG TEST(NSG, Query) { auto index_type = knowhere::IndexEnum::INDEX_NSG; - auto metric_type = knowhere::Metric::L2; + auto metric_type = knowhere::metric::L2; auto conf = generate_conf(index_type, metric_type); auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto dataset = GenDataset(NB, metric_type, false); @@ -185,7 +184,7 @@ TEST(NSG, Query) { TEST(BINFLAT, Build) { auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT; - auto metric_type = knowhere::Metric::JACCARD; + auto metric_type = knowhere::metric::JACCARD; auto conf = generate_conf(index_type, metric_type); auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto dataset = GenDataset(NB, metric_type, true); @@ -212,11 +211,11 @@ TEST(BinIVFFlat, Build_and_Query) { knowhere::KnowhereConfig::SetStatisticsLevel(2); auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT; - auto metric_type = knowhere::Metric::TANIMOTO; + auto metric_type = knowhere::metric::TANIMOTO; auto conf = generate_conf(index_type, metric_type); auto topk = 10; - conf[knowhere::meta::TOPK] = topk; - conf[knowhere::IndexParams::nlist] = 1; + knowhere::SetMetaTopk(conf, topk); + knowhere::SetIndexParamNlist(conf, 1); auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto nb = 2; auto dim = 128; @@ -231,8 +230,8 @@ TEST(BinIVFFlat, Build_and_Query) { auto xq_dataset = knowhere::GenDataset(nq, dim, xq_data.data()); auto result = index->Query(xq_dataset, conf, nullptr); - auto hit_ids = result->Get(knowhere::meta::IDS); - auto distances = result->Get(knowhere::meta::DISTANCE); + auto hit_ids = knowhere::GetDatasetIDs(result); + auto distances = knowhere::GetDatasetDistance(result); auto query_res = std::make_unique(); query_res->nq = nq; @@ -244,8 +243,7 @@ TEST(BinIVFFlat, Build_and_Query) { print_query_result(query_res); - index->UpdateIndexSize(); - ASSERT_GT(index->IndexSize(), 0); + ASSERT_GT(index->Size(), 0); auto stats = index->GetStatistics(); ASSERT_TRUE(stats != nullptr); @@ -254,7 +252,7 @@ TEST(BinIVFFlat, Build_and_Query) { TEST(BINIDMAP, Build) { auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP; - auto metric_type = knowhere::Metric::JACCARD; + auto metric_type = knowhere::metric::JACCARD; auto conf = generate_conf(index_type, metric_type); auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type); auto dataset = GenDataset(NB, metric_type, true); @@ -267,7 +265,7 @@ TEST(BINIDMAP, Build) { TEST(PQWrapper, Build) { auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFPQ; - auto metric_type = knowhere::Metric::L2; + auto metric_type = knowhere::metric::L2; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -287,7 +285,7 @@ TEST(PQWrapper, Build) { TEST(IVFFLATNMWrapper, Build) { auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; - auto metric_type = knowhere::Metric::L2; + auto metric_type = knowhere::metric::L2; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -308,7 +306,7 @@ TEST(IVFFLATNMWrapper, Build) { TEST(IVFFLATNMWrapper, Codec) { int64_t flat_nb = 100000; auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; - auto metric_type = knowhere::Metric::L2; + auto metric_type = knowhere::metric::L2; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -342,7 +340,7 @@ TEST(IVFFLATNMWrapper, Codec) { TEST(BinFlatWrapper, Build) { auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT; - auto metric_type = knowhere::Metric::JACCARD; + auto metric_type = knowhere::metric::JACCARD; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -365,7 +363,7 @@ TEST(BinFlatWrapper, Build) { TEST(BinIdMapWrapper, Build) { auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP; - auto metric_type = knowhere::Metric::JACCARD; + auto metric_type = knowhere::metric::JACCARD; indexcgo::TypeParams type_params; indexcgo::IndexParams index_params; std::tie(type_params, index_params) = generate_params(index_type, metric_type); @@ -389,28 +387,28 @@ TEST(BinIdMapWrapper, Build) { INSTANTIATE_TEST_CASE_P( IndexTypeParameters, IndexWrapperTest, - ::testing::Values(std::pair(knowhere::IndexEnum::INDEX_FAISS_IDMAP, knowhere::Metric::L2), - std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFPQ, knowhere::Metric::L2), - std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, knowhere::Metric::L2), - std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, knowhere::Metric::L2), - std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::Metric::JACCARD), - std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::Metric::TANIMOTO), - std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, knowhere::Metric::JACCARD), + ::testing::Values(std::pair(knowhere::IndexEnum::INDEX_FAISS_IDMAP, knowhere::metric::L2), + std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFPQ, knowhere::metric::L2), + std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, knowhere::metric::L2), + std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, knowhere::metric::L2), + std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::metric::JACCARD), + std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::metric::TANIMOTO), + std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, knowhere::metric::JACCARD), #ifdef MILVUS_SUPPORT_SPTAG - std::pair(knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, knowhere::Metric::L2), - std::pair(knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, knowhere::metric::L2), + std::pair(knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, knowhere::metric::L2), #endif - std::pair(knowhere::IndexEnum::INDEX_HNSW, knowhere::Metric::L2), - std::pair(knowhere::IndexEnum::INDEX_ANNOY, knowhere::Metric::L2), - std::pair(knowhere::IndexEnum::INDEX_RHNSWFlat, knowhere::Metric::L2), - std::pair(knowhere::IndexEnum::INDEX_RHNSWPQ, knowhere::Metric::L2), - std::pair(knowhere::IndexEnum::INDEX_RHNSWSQ, knowhere::Metric::L2) + std::pair(knowhere::IndexEnum::INDEX_HNSW, knowhere::metric::L2), + std::pair(knowhere::IndexEnum::INDEX_ANNOY, knowhere::metric::L2), + std::pair(knowhere::IndexEnum::INDEX_RHNSWFlat, knowhere::metric::L2), + std::pair(knowhere::IndexEnum::INDEX_RHNSWPQ, knowhere::metric::L2), + std::pair(knowhere::IndexEnum::INDEX_RHNSWSQ, knowhere::metric::L2) #ifdef MILVUS_SUPPORT_NGT - std::pair(knowhere::IndexEnum::INDEX_NGTPANNG, knowhere::Metric::L2), - std::pair(knowhere::IndexEnum::INDEX_NGTONNG, knowhere::Metric::L2), + std::pair(knowhere::IndexEnum::INDEX_NGTPANNG, knowhere::metric::L2), + std::pair(knowhere::IndexEnum::INDEX_NGTONNG, knowhere::metric::L2), #endif #ifdef MILVUS_SUPPORT_NSG - std::pair(knowhere::IndexEnum::INDEX_NSG, knowhere::Metric::L2) + std::pair(knowhere::IndexEnum::INDEX_NSG, knowhere::metric::L2) #endif )); diff --git a/internal/core/unittest/test_indexing.cpp b/internal/core/unittest/test_indexing.cpp index be1c8f1727..f19b8f3919 100644 --- a/internal/core/unittest/test_indexing.cpp +++ b/internal/core/unittest/test_indexing.cpp @@ -144,14 +144,14 @@ TEST(Indexing, Naive) { knowhere::IndexMode::MODE_CPU); auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 4}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 4}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, + {knowhere::meta::DEVICE_ID, 0}, }; // auto ds = knowhere::GenDataset(N, DIM, raw_data.data()); @@ -199,8 +199,8 @@ TEST(Indexing, Naive) { BitsetView view = bitmap; auto query_ds = knowhere::GenDataset(1, DIM, raw_data.data()); auto final = index->Query(query_ds, conf, view); - auto ids = final->Get(knowhere::meta::IDS); - auto distances = final->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(final); + auto distances = knowhere::GetDatasetDistance(final); for (int i = 0; i < TOPK; ++i) { if (ids[i] < N / 2) { std::cout << "WRONG: "; @@ -221,12 +221,14 @@ TEST(Indexing, IVFFlat) { auto [raw_data, timestamps, uids] = generate_data(N); std::cout << "generate data: " << timer.get_step_seconds() << " seconds" << std::endl; auto indexing = std::make_shared(); - auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, - {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, NLIST}, - {knowhere::IndexParams::nprobe, NPROBE}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::indexparam::NLIST, NLIST}, + {knowhere::indexparam::NPROBE, NPROBE}, + {knowhere::meta::DEVICE_ID, 0} + }; auto database = knowhere::GenDataset(N, DIM, raw_data.data()); std::cout << "init ivf " << timer.get_step_seconds() << " seconds" << std::endl; @@ -242,8 +244,8 @@ TEST(Indexing, IVFFlat) { auto result = indexing->Query(dataset, conf, nullptr); std::cout << "query ivf " << timer.get_step_seconds() << " seconds" << std::endl; - auto ids = result->Get(knowhere::meta::IDS); - auto dis = result->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result); + auto dis = knowhere::GetDatasetDistance(result); for (int i = 0; i < std::min(NQ * TOPK, 100); ++i) { std::cout << ids[i] << "->" << dis[i] << std::endl; } @@ -261,12 +263,14 @@ TEST(Indexing, IVFFlatNM) { auto [raw_data, timestamps, uids] = generate_data(N); std::cout << "generate data: " << timer.get_step_seconds() << " seconds" << std::endl; auto indexing = std::make_shared(); - auto conf = knowhere::Config{{knowhere::meta::DIM, DIM}, - {knowhere::meta::TOPK, TOPK}, - {knowhere::IndexParams::nlist, NLIST}, - {knowhere::IndexParams::nprobe, NPROBE}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, TOPK}, + {knowhere::indexparam::NLIST, NLIST}, + {knowhere::indexparam::NPROBE, NPROBE}, + {knowhere::meta::DEVICE_ID, 0} + }; auto database = knowhere::GenDataset(N, DIM, raw_data.data()); std::cout << "init ivf_nm " << timer.get_step_seconds() << " seconds" << std::endl; @@ -275,7 +279,6 @@ TEST(Indexing, IVFFlatNM) { indexing->AddWithoutIds(database, conf); std::cout << "insert ivf_nm " << timer.get_step_seconds() << " seconds" << std::endl; - indexing->SetIndexSize(NQ * DIM * sizeof(float)); knowhere::BinarySet bs = indexing->Serialize(conf); knowhere::BinaryPtr bptr = std::make_shared(); @@ -291,8 +294,8 @@ TEST(Indexing, IVFFlatNM) { auto result = indexing->Query(dataset, conf, nullptr); std::cout << "query ivf_nm " << timer.get_step_seconds() << " seconds" << std::endl; - auto ids = result->Get(knowhere::meta::IDS); - auto dis = result->Get(knowhere::meta::DISTANCE); + auto ids = knowhere::GetDatasetIDs(result); + auto dis = knowhere::GetDatasetDistance(result); for (int i = 0; i < std::min(NQ * TOPK, 100); ++i) { std::cout << ids[i] << "->" << dis[i] << std::endl; } diff --git a/internal/core/unittest/test_scalar_index_creator.cpp b/internal/core/unittest/test_scalar_index_creator.cpp index 9f98a92479..49ece94755 100644 --- a/internal/core/unittest/test_scalar_index_creator.cpp +++ b/internal/core/unittest/test_scalar_index_creator.cpp @@ -54,7 +54,7 @@ build_index(const ScalarIndexCreatorPtr& creator, const std::vector& arr) creator->Build(ds); - delete[](char*) ds->Get(knowhere::meta::TENSOR); + delete[](char*) knowhere::GetDatasetTensor(ds); } template <> @@ -66,7 +66,7 @@ build_index(const ScalarIndexCreatorPtr& creator, const std::vector creator->Build(ds); - delete[](char*) ds->Get(knowhere::meta::TENSOR); + delete[](char*) knowhere::GetDatasetTensor(ds); } } // namespace diff --git a/internal/core/unittest/test_sealed.cpp b/internal/core/unittest/test_sealed.cpp index fa05acda50..662b7d87a7 100644 --- a/internal/core/unittest/test_sealed.cpp +++ b/internal/core/unittest/test_sealed.cpp @@ -11,8 +11,8 @@ #include +#include "knowhere/index/VecIndex.h" #include "knowhere/index/vector_index/IndexIVF.h" -#include "knowhere/index/vector_index/VecIndex.h" #include "knowhere/index/vector_index/adapter/VectorAdapter.h" #include "segcore/SegmentSealedImpl.h" #include "test_utils/DataGen.h" @@ -78,12 +78,14 @@ TEST(Sealed, without_predicate) { auto pre_result = SearchResultToJson(*sr); auto indexing = std::make_shared(); - auto conf = knowhere::Config{{knowhere::meta::DIM, dim}, - {knowhere::meta::TOPK, topK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, dim}, + {knowhere::meta::TOPK, topK}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::meta::DEVICE_ID, 0} + }; auto database = knowhere::GenDataset(N, dim, vec_col.data() + 1000 * dim); indexing->Train(database, conf); @@ -96,8 +98,8 @@ TEST(Sealed, without_predicate) { auto result = indexing->Query(query_dataset, conf, nullptr); - auto ids = result->Get(knowhere::meta::IDS); // for comparison - auto dis = result->Get(knowhere::meta::DISTANCE); // for comparison + auto ids = knowhere::GetDatasetIDs(result); // for comparison + auto dis = knowhere::GetDatasetDistance(result); // for comparison std::vector vec_ids(ids, ids + topK * num_queries); std::vector vec_dis(dis, dis + topK * num_queries); @@ -183,12 +185,14 @@ TEST(Sealed, with_predicate) { auto sr = segment->Search(plan.get(), ph_group.get(), time); auto indexing = std::make_shared(); - auto conf = knowhere::Config{{knowhere::meta::DIM, dim}, - {knowhere::meta::TOPK, topK}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 10}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, dim}, + {knowhere::meta::TOPK, topK}, + {knowhere::indexparam::NLIST, 100}, + {knowhere::indexparam::NPROBE, 10}, + {knowhere::meta::DEVICE_ID, 0} + }; auto database = knowhere::GenDataset(N, dim, vec_col.data()); indexing->Train(database, conf); @@ -286,7 +290,7 @@ TEST(Sealed, LoadFieldData) { LoadIndexInfo vec_info; vec_info.field_id = fakevec_id.get(); vec_info.index = indexing; - vec_info.index_params["metric_type"] = knowhere::Metric::L2; + vec_info.index_params["metric_type"] = knowhere::metric::L2; segment->LoadIndex(vec_info); ASSERT_EQ(segment->num_chunk(), 1); @@ -414,7 +418,7 @@ TEST(Sealed, LoadScalarIndex) { vec_info.field_id = fakevec_id.get(); vec_info.field_type = CDataType::FloatVector; vec_info.index = indexing; - vec_info.index_params["metric_type"] = knowhere::Metric::L2; + vec_info.index_params["metric_type"] = knowhere::metric::L2; segment->LoadIndex(vec_info); LoadIndexInfo counter_index; diff --git a/internal/core/unittest/test_string_index.cpp b/internal/core/unittest/test_string_index.cpp index 78e461fa85..9b20d3f9a5 100644 --- a/internal/core/unittest/test_string_index.cpp +++ b/internal/core/unittest/test_string_index.cpp @@ -39,7 +39,7 @@ class StringIndexBaseTest : public ::testing::Test { void TearDown() override { - delete[](char*)(str_ds->Get(knowhere::meta::TENSOR)); + delete[](char*)(knowhere::GetDatasetTensor(str_ds)); } protected: diff --git a/internal/core/unittest/test_utils/DataGen.h b/internal/core/unittest/test_utils/DataGen.h index cdde383930..ed083f9a3c 100644 --- a/internal/core/unittest/test_utils/DataGen.h +++ b/internal/core/unittest/test_utils/DataGen.h @@ -15,20 +15,20 @@ #include #include #include -#include "google/protobuf/text_format.h" -#include -#include -#include -#include +#include #include "Constants.h" #include "common/Schema.h" +#include "index/ScalarIndexSort.h" +#include "index/StringIndexSort.h" +#include "knowhere/index/VecIndex.h" +#include "knowhere/index/VecIndexFactory.h" +#include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/index/vector_index/adapter/VectorAdapter.h" #include "query/SearchOnIndex.h" #include "segcore/SegmentGrowingImpl.h" #include "segcore/SegmentSealedImpl.h" #include "segcore/Utils.h" -#include "index/ScalarIndexSort.h" -#include "index/StringIndexSort.h" using boost::algorithm::starts_with; @@ -396,10 +396,12 @@ SealedCreator(SchemaPtr schema, const GeneratedData& dataset) { inline knowhere::VecIndexPtr GenVecIndexing(int64_t N, int64_t dim, const float* vec) { // {knowhere::IndexParams::nprobe, 10}, - auto conf = knowhere::Config{{knowhere::meta::DIM, dim}, - {knowhere::IndexParams::nlist, 1024}, - {knowhere::Metric::TYPE, knowhere::Metric::L2}, - {knowhere::meta::DEVICEID, 0}}; + auto conf = knowhere::Config{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::L2}, + {knowhere::meta::DIM, dim}, + {knowhere::indexparam::NLIST, 1024}, + {knowhere::meta::DEVICE_ID, 0} + }; auto database = knowhere::GenDataset(N, dim, vec); auto indexing = std::make_shared(); indexing->Train(database, conf); diff --git a/internal/core/unittest/test_utils/indexbuilder_test_utils.h b/internal/core/unittest/test_utils/indexbuilder_test_utils.h index 8c724284a9..380662d950 100644 --- a/internal/core/unittest/test_utils/indexbuilder_test_utils.h +++ b/internal/core/unittest/test_utils/indexbuilder_test_utils.h @@ -16,21 +16,20 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include "pb/index_cgo_msg.pb.h" - -#include "indexbuilder/VecIndexCreator.h" -#include "indexbuilder/index_c.h" #include "DataGen.h" -#include "indexbuilder/utils.h" -#include "indexbuilder/helper.h" +#include "index/ScalarIndex.h" +#include "index/StringIndex.h" #include "indexbuilder/ScalarIndexCreator.h" +#include "indexbuilder/VecIndexCreator.h" +#include "indexbuilder/helper.h" +#include "indexbuilder/index_c.h" +#include "indexbuilder/utils.h" +#include "knowhere/common/MetricType.h" +#include "knowhere/index/VecIndexFactory.h" +#include "knowhere/index/vector_index/helpers/IndexParameter.h" +#include "knowhere/index/vector_index/adapter/VectorAdapter.h" +#include "pb/index_cgo_msg.pb.h" constexpr int64_t DIM = 8; constexpr int64_t NQ = 10; @@ -52,164 +51,169 @@ auto generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& metric_type) { if (index_type == knowhere::IndexEnum::INDEX_FAISS_IDMAP) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, }; } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::IndexParams::nlist, 16}, - {knowhere::IndexParams::nprobe, 4}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, + {knowhere::indexparam::NLIST, 16}, + {knowhere::indexparam::NPROBE, 4}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, }; } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::IndexParams::nlist, 16}, - {knowhere::IndexParams::nprobe, 4}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, + {knowhere::indexparam::NLIST, 16}, + {knowhere::indexparam::NPROBE, 4}, #ifdef MILVUS_GPU_VERSION - {knowhere::meta::DEVICEID, DEVICEID}, + {knowhere::meta::DEVICE_ID, DEVICEID}, #endif }; } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::IndexParams::nlist, 16}, - {knowhere::IndexParams::nprobe, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, + {knowhere::indexparam::NLIST, 16}, + {knowhere::indexparam::NPROBE, 4}, + {knowhere::indexparam::NBITS, 8}, #ifdef MILVUS_GPU_VERSION - {knowhere::meta::DEVICEID, DEVICEID}, + {knowhere::meta::DEVICE_ID, DEVICEID}, #endif }; } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::IndexParams::nlist, 16}, - {knowhere::IndexParams::nprobe, 4}, - {knowhere::IndexParams::m, 4}, - {knowhere::IndexParams::nbits, 8}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, + {knowhere::indexparam::NLIST, 16}, + {knowhere::indexparam::NPROBE, 4}, + {knowhere::indexparam::M, 4}, + {knowhere::indexparam::NBITS, 8}, }; } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::Metric::TYPE, metric_type}, }; #ifdef MILVUS_SUPPORT_NSG } else if (index_type == knowhere::IndexEnum::INDEX_NSG) { return knowhere::Config{ + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, - {knowhere::IndexParams::nlist, 163}, {knowhere::meta::TOPK, K}, - {knowhere::IndexParams::nprobe, 8}, - {knowhere::IndexParams::knng, 20}, - {knowhere::IndexParams::search_length, 40}, - {knowhere::IndexParams::out_degree, 30}, - {knowhere::IndexParams::candidate, 100}, - {knowhere::Metric::TYPE, metric_type}, + {knowhere::indexparam::NLIST, 163}, + {knowhere::indexparam::NPROBE, 8}, + {knowhere::indexparam::KNNG, 20}, + {knowhere::indexparam::SEARCH_LENGTH, 40}, + {knowhere::indexparam::OUT_DEGREE, 30}, + {knowhere::indexparam::CANDIDATE, 100}, }; #endif #ifdef MILVUS_SUPPORT_SPTAG } else if (index_type == knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, - // {knowhere::meta::TOPK, 10}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, + {knowhere::meta::TOPK, 10}, }; } else if (index_type == knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, - // {knowhere::meta::TOPK, 10}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, + {knowhere::meta::TOPK, 10}, }; #endif } else if (index_type == knowhere::IndexEnum::INDEX_HNSW) { return knowhere::Config{ - {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::IndexParams::M, 16}, {knowhere::IndexParams::efConstruction, 200}, - {knowhere::IndexParams::ef, 200}, {knowhere::Metric::TYPE, metric_type}, + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, + {knowhere::meta::DIM, DIM}, + {knowhere::meta::TOPK, K}, + {knowhere::indexparam::HNSW_M, 16}, + {knowhere::indexparam::EFCONSTRUCTION, 200}, + {knowhere::indexparam::EF, 200}, }; } else if (index_type == knowhere::IndexEnum::INDEX_ANNOY) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::IndexParams::n_trees, 4}, - {knowhere::IndexParams::search_k, 100}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, + {knowhere::indexparam::N_TREES, 4}, + {knowhere::indexparam::SEARCH_K, 100}, }; } else if (index_type == knowhere::IndexEnum::INDEX_RHNSWFlat) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::IndexParams::M, 16}, - {knowhere::IndexParams::efConstruction, 200}, - {knowhere::IndexParams::ef, 200}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, + {knowhere::indexparam::HNSW_M, 16}, + {knowhere::indexparam::EFCONSTRUCTION, 200}, + {knowhere::indexparam::EF, 200}, }; } else if (index_type == knowhere::IndexEnum::INDEX_RHNSWPQ) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::IndexParams::M, 16}, - {knowhere::IndexParams::efConstruction, 200}, - {knowhere::IndexParams::ef, 200}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, - {knowhere::IndexParams::PQM, 8}, + {knowhere::indexparam::HNSW_M, 16}, + {knowhere::indexparam::EFCONSTRUCTION, 200}, + {knowhere::indexparam::EF, 200}, + {knowhere::indexparam::PQ_M, 8}, }; } else if (index_type == knowhere::IndexEnum::INDEX_RHNSWSQ) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::IndexParams::M, 16}, - {knowhere::IndexParams::efConstruction, 200}, - {knowhere::IndexParams::ef, 200}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, + {knowhere::indexparam::HNSW_M, 16}, + {knowhere::indexparam::EFCONSTRUCTION, 200}, + {knowhere::indexparam::EF, 200}, }; #ifdef MILVUS_SUPPORT_NGT } else if (index_type == knowhere::IndexEnum::INDEX_NGTPANNG) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::IndexParams::edge_size, 10}, - {knowhere::IndexParams::epsilon, 0.1}, - {knowhere::IndexParams::max_search_edges, 50}, - {knowhere::IndexParams::forcedly_pruned_edge_size, 60}, - {knowhere::IndexParams::selectively_pruned_edge_size, 30}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, + {knowhere::indexparam::EDGE_SIZE, 10}, + {knowhere::indexparam::EPSILON, 0.1}, + {knowhere::indexparam::MAX_SEARCH_EDGES, 50}, + {knowhere::indexparam::FORCEDLY_PRUNED_EDGE_SIZE, 60}, + {knowhere::indexparam::SELECTIVELY_PRUNED_EDGE_SIZE, 30}, }; } else if (index_type == knowhere::IndexEnum::INDEX_NGTONNG) { return knowhere::Config{ + {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size}, + {knowhere::meta::METRIC_TYPE, metric_type}, {knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K}, - {knowhere::Metric::TYPE, metric_type}, - {knowhere::IndexParams::edge_size, 20}, - {knowhere::IndexParams::epsilon, 0.1}, - {knowhere::IndexParams::max_search_edges, 50}, - {knowhere::IndexParams::outgoing_edge_size, 5}, - {knowhere::IndexParams::incoming_edge_size, 40}, - {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size}, + {knowhere::indexparam::EDGE_SIZE, 20}, + {knowhere::indexparam::EPSILON, 0.1}, + {knowhere::indexparam::MAX_SEARCH_EDGES, 50}, + {knowhere::indexparam::OUTGOING_EDGE_SIZE, 5}, + {knowhere::indexparam::INCOMING_EDGE_SIZE, 40}, }; #endif } @@ -241,7 +245,7 @@ generate_params(const knowhere::IndexType& index_type, const knowhere::MetricTyp auto GenDataset(int64_t N, const knowhere::MetricType& metric_type, bool is_binary, int64_t dim = DIM) { auto schema = std::make_shared(); - auto faiss_metric_type = knowhere::GetMetricType(metric_type); + auto faiss_metric_type = knowhere::GetMetricType(std::string(metric_type)); if (!is_binary) { schema->AddDebugField("fakevec", milvus::DataType::VECTOR_FLOAT, dim, faiss_metric_type); return milvus::segcore::DataGen(schema, N); @@ -312,9 +316,9 @@ CountDistance( if (point_a == nullptr || point_b == nullptr) { return std::numeric_limits::max(); } - if (metric == knowhere::Metric::L2) { + if (metric == knowhere::metric::L2) { return L2(static_cast(point_a), static_cast(point_b), dim); - } else if (metric == knowhere::Metric::JACCARD) { + } else if (metric == knowhere::metric::JACCARD) { return Jaccard(static_cast(point_a), static_cast(point_b), dim); } else { return std::numeric_limits::max(); @@ -327,9 +331,9 @@ CheckDistances(const QueryResultPtr& result, const knowhere::DatasetPtr& query_dataset, const knowhere::MetricType& metric, const float threshold = 1.0e-5) { - auto base_vecs = base_dataset->Get(knowhere::meta::TENSOR); - auto query_vecs = query_dataset->Get(knowhere::meta::TENSOR); - auto dim = base_dataset->Get(knowhere::meta::DIM); + auto base_vecs = (float*)knowhere::GetDatasetTensor(base_dataset); + auto query_vecs = (float*)knowhere::GetDatasetTensor(query_dataset); + auto dim = knowhere::GetDatasetDim(base_dataset); auto nq = result->nq; auto k = result->topk; for (auto i = 0; i < nq; i++) {