diff --git a/configs/milvus.yaml b/configs/milvus.yaml index 689e5c6b26..b701f2d741 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -413,6 +413,7 @@ queryNode: nprobe: 16 # nprobe to search small index, based on your accuracy requirement, must smaller than nlist subDim: 4 # interim index sub dim, recommend to (subDim % vector dim == 0) refineRatio: 4.5 # interim index parameters, should set to be >= 1.0 + indexBuildRatio: 0.1 # the ratio of building interim index rows count with max row count of a flush segment, should set to be < 1.0 refineQuantType: NONE # Data representation of SCANN_DVR index, options: 'NONE', 'FLOAT16', 'BFLOAT16' and 'UINT8' refineWithQuant: true # whether to use refineQuantType to refine for faster but loss a little precision denseVectorIndexType: IVF_FLAT_CC # Dense vector intermin index type diff --git a/internal/core/src/segcore/IndexConfigGenerator.cpp b/internal/core/src/segcore/IndexConfigGenerator.cpp index b12735cb15..8ee7416cb6 100644 --- a/internal/core/src/segcore/IndexConfigGenerator.cpp +++ b/internal/core/src/segcore/IndexConfigGenerator.cpp @@ -93,8 +93,7 @@ VecIndexConfig::GetBuildThreshold() const noexcept { if (is_sparse_) { return 0; } - assert(VecIndexConfig::index_build_ratio.count(index_type_)); - auto ratio = VecIndexConfig::index_build_ratio.at(index_type_); + auto ratio = config_.get_build_ratio(); assert(ratio >= 0.0 && ratio < 1.0); return std::max(int64_t(max_index_row_count_ * ratio), config_.get_nlist() * 39); diff --git a/internal/core/src/segcore/IndexConfigGenerator.h b/internal/core/src/segcore/IndexConfigGenerator.h index be7f738cdf..2af089a8f5 100644 --- a/internal/core/src/segcore/IndexConfigGenerator.h +++ b/internal/core/src/segcore/IndexConfigGenerator.h @@ -30,10 +30,6 @@ enum class IndexConfigLevel { // this is the config used for generating growing index or the temp sealed index // when the segment is sealed before the index is built. class VecIndexConfig { - inline static const std::map index_build_ratio = { - {knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC, 0.1}, - {knowhere::IndexEnum::INDEX_FAISS_SCANN_DVR, 0.1}}; - inline static const std::unordered_set maintain_params = { "radius", "range_filter", "drop_ratio_search", "dim_max_score_ratio"}; diff --git a/internal/core/src/segcore/SegcoreConfig.h b/internal/core/src/segcore/SegcoreConfig.h index 8005bf2a33..2887eefb99 100644 --- a/internal/core/src/segcore/SegcoreConfig.h +++ b/internal/core/src/segcore/SegcoreConfig.h @@ -88,6 +88,16 @@ class SegcoreConfig { refine_ratio_ = refine_ratio; } + void + set_build_ratio(float build_ratio) { + build_ratio_ = build_ratio; + } + + float + get_build_ratio() const { + return build_ratio_; + } + int64_t get_refine_ratio() const { return refine_ratio_; @@ -149,6 +159,7 @@ class SegcoreConfig { inline static int64_t nprobe_ = 4; inline static int64_t sub_dim_ = 2; inline static float refine_ratio_ = 3.0; + inline static float build_ratio_ = 0.1; inline static std::string dense_index_type_ = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC; inline static knowhere::RefineType refine_type_ = diff --git a/internal/core/src/segcore/segcore_init_c.cpp b/internal/core/src/segcore/segcore_init_c.cpp index 19cc28a51a..c076ee59c7 100644 --- a/internal/core/src/segcore/segcore_init_c.cpp +++ b/internal/core/src/segcore/segcore_init_c.cpp @@ -105,6 +105,13 @@ SegcoreSetRefineRatio(const float value) { config.set_refine_ratio(value); } +extern "C" void +SegcoreSetIndexBuildRatio(const float value) { + milvus::segcore::SegcoreConfig& config = + milvus::segcore::SegcoreConfig::default_config(); + config.set_build_ratio(value); +} + extern "C" void SegcoreSetKnowhereBuildThreadPoolNum(const uint32_t num_threads) { milvus::config::KnowhereInitBuildThreadPool(num_threads); diff --git a/internal/core/src/segcore/segcore_init_c.h b/internal/core/src/segcore/segcore_init_c.h index 66e76e729d..7836fbdb27 100644 --- a/internal/core/src/segcore/segcore_init_c.h +++ b/internal/core/src/segcore/segcore_init_c.h @@ -43,6 +43,10 @@ SegcoreSetSubDim(const int64_t); void SegcoreSetRefineRatio(const float); +void +SegcoreSetIndexBuildRatio(const float); + + void SegcoreInterminDenseIndexType(const char*); diff --git a/internal/util/initcore/init_core.go b/internal/util/initcore/init_core.go index 18b2dbf0d6..83497329e3 100644 --- a/internal/util/initcore/init_core.go +++ b/internal/util/initcore/init_core.go @@ -246,6 +246,9 @@ func InitInterminIndexConfig(params *paramtable.ComponentParam) error { refineRatio := C.float(params.QueryNodeCfg.InterimIndexRefineRatio.GetAsFloat()) C.SegcoreSetRefineRatio(refineRatio) + indexBuildRatio := C.float(params.QueryNodeCfg.InterimIndexBuildRatio.GetAsFloat()) + C.SegcoreSetIndexBuildRatio(indexBuildRatio) + denseVecIndexType := C.CString(params.QueryNodeCfg.DenseVectorInterminIndexType.GetValue()) defer C.free(unsafe.Pointer(denseVecIndexType)) status := C.SegcoreSetDenseVectorInterminIndexType(denseVecIndexType) diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 0be0d9f30b..844a2a996b 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -2709,6 +2709,7 @@ type queryNodeConfig struct { InterimIndexNProbe ParamItem `refreshable:"false"` InterimIndexSubDim ParamItem `refreshable:"false"` InterimIndexRefineRatio ParamItem `refreshable:"false"` + InterimIndexBuildRatio ParamItem `refreshable:"false"` InterimIndexRefineQuantType ParamItem `refreshable:"false"` InterimIndexRefineWithQuant ParamItem `refreshable:"false"` DenseVectorInterminIndexType ParamItem `refreshable:"false"` @@ -3036,6 +3037,21 @@ This defaults to true, indicating that Milvus creates temporary index for growin } p.InterimIndexRefineRatio.Init(base.mgr) + p.InterimIndexBuildRatio = ParamItem{ + Key: "queryNode.segcore.interimIndex.indexBuildRatio", + Version: "2.5.18", + Formatter: func(v string) string { + if getAsFloat(v) > 1.0 { + return "0.1" + } + return v + }, + DefaultValue: "0.1", + Doc: "the ratio of building interim index rows count with max row count of a flush segment, should set to be < 1.0", + Export: true, + } + p.InterimIndexBuildRatio.Init(base.mgr) + p.LoadMemoryUsageFactor = ParamItem{ Key: "queryNode.loadMemoryUsageFactor", Version: "2.0.0",