enhance:[2.5] Make build ratio of interim index configurable (#43938)

issue: https://github.com/milvus-io/milvus/issues/43993
master pr: https://github.com/milvus-io/milvus/pull/43939

Signed-off-by: cqy123456 <qianya.cheng@zilliz.com>

configs/milvus.yaml

@@ -413,6 +413,7 @@ queryNode:
       nprobe: 16 # nprobe to search small index, based on your accuracy requirement, must smaller than nlist
       subDim: 4 # interim index sub dim, recommend to (subDim % vector dim == 0)
       refineRatio: 4.5 # interim index parameters, should set to be >= 1.0
+      indexBuildRatio: 0.1 # the ratio of building interim index rows count with max row count of a flush segment, should set to be < 1.0
       refineQuantType: NONE # Data representation of SCANN_DVR index, options: 'NONE', 'FLOAT16', 'BFLOAT16' and 'UINT8'
       refineWithQuant: true # whether to use refineQuantType to refine for faster but loss a little precision
       denseVectorIndexType: IVF_FLAT_CC # Dense vector intermin index type

internal/core/src/segcore/IndexConfigGenerator.cpp

@@ -93,8 +93,7 @@ VecIndexConfig::GetBuildThreshold() const noexcept {
     if (is_sparse_) {
         return 0;
     }
-    assert(VecIndexConfig::index_build_ratio.count(index_type_));
+    auto ratio = config_.get_build_ratio();
-    auto ratio = VecIndexConfig::index_build_ratio.at(index_type_);
     assert(ratio >= 0.0 && ratio < 1.0);
     return std::max(int64_t(max_index_row_count_ * ratio),
                     config_.get_nlist() * 39);

internal/core/src/segcore/IndexConfigGenerator.h

@@ -30,10 +30,6 @@ enum class IndexConfigLevel {
 // this is the config used for generating growing index or the temp sealed index
 // when the segment is sealed before the index is built.
 class VecIndexConfig {
-    inline static const std::map<std::string, double> index_build_ratio = {
-        {knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC, 0.1},
-        {knowhere::IndexEnum::INDEX_FAISS_SCANN_DVR, 0.1}};
-
     inline static const std::unordered_set<std::string> maintain_params = {
         "radius", "range_filter", "drop_ratio_search", "dim_max_score_ratio"};
 

internal/core/src/segcore/SegcoreConfig.h

@@ -88,6 +88,16 @@ class SegcoreConfig {
         refine_ratio_ = refine_ratio;
     }
 
+    void
+    set_build_ratio(float build_ratio) {
+        build_ratio_ = build_ratio;
+    }
+
+    float 
+    get_build_ratio() const {
+        return build_ratio_;
+    }
+
     int64_t
     get_refine_ratio() const {
         return refine_ratio_;
@@ -149,6 +159,7 @@ class SegcoreConfig {
     inline static int64_t nprobe_ = 4;
     inline static int64_t sub_dim_ = 2;
     inline static float refine_ratio_ = 3.0;
+    inline static float build_ratio_  = 0.1;
     inline static std::string dense_index_type_ =
         knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC;
     inline static knowhere::RefineType refine_type_ =

internal/core/src/segcore/segcore_init_c.cpp

@@ -105,6 +105,13 @@ SegcoreSetRefineRatio(const float value) {
     config.set_refine_ratio(value);
 }
 
+extern "C" void
+SegcoreSetIndexBuildRatio(const float value) {
+    milvus::segcore::SegcoreConfig& config = 
+        milvus::segcore::SegcoreConfig::default_config();
+    config.set_build_ratio(value);
+}
+
 extern "C" void
 SegcoreSetKnowhereBuildThreadPoolNum(const uint32_t num_threads) {
     milvus::config::KnowhereInitBuildThreadPool(num_threads);

internal/core/src/segcore/segcore_init_c.h

@@ -43,6 +43,10 @@ SegcoreSetSubDim(const int64_t);
 void
 SegcoreSetRefineRatio(const float);
 
+void
+SegcoreSetIndexBuildRatio(const float);
+
+
 void
 SegcoreInterminDenseIndexType(const char*);
 

internal/util/initcore/init_core.go

@@ -246,6 +246,9 @@ func InitInterminIndexConfig(params *paramtable.ComponentParam) error {
 	refineRatio := C.float(params.QueryNodeCfg.InterimIndexRefineRatio.GetAsFloat())
 	C.SegcoreSetRefineRatio(refineRatio)
 
+	indexBuildRatio := C.float(params.QueryNodeCfg.InterimIndexBuildRatio.GetAsFloat())
+	C.SegcoreSetIndexBuildRatio(indexBuildRatio)
+
 	denseVecIndexType := C.CString(params.QueryNodeCfg.DenseVectorInterminIndexType.GetValue())
 	defer C.free(unsafe.Pointer(denseVecIndexType))
 	status := C.SegcoreSetDenseVectorInterminIndexType(denseVecIndexType)

pkg/util/paramtable/component_param.go

@@ -2709,6 +2709,7 @@ type queryNodeConfig struct {
 	InterimIndexNProbe            ParamItem `refreshable:"false"`
 	InterimIndexSubDim            ParamItem `refreshable:"false"`
 	InterimIndexRefineRatio       ParamItem `refreshable:"false"`
+	InterimIndexBuildRatio        ParamItem `refreshable:"false"`
 	InterimIndexRefineQuantType   ParamItem `refreshable:"false"`
 	InterimIndexRefineWithQuant   ParamItem `refreshable:"false"`
 	DenseVectorInterminIndexType  ParamItem `refreshable:"false"`
@@ -3036,6 +3037,21 @@ This defaults to true, indicating that Milvus creates temporary index for growin
 	}
 	p.InterimIndexRefineRatio.Init(base.mgr)
 
+	p.InterimIndexBuildRatio = ParamItem{
+		Key:     "queryNode.segcore.interimIndex.indexBuildRatio",
+		Version: "2.5.18",
+		Formatter: func(v string) string {
+			if getAsFloat(v) > 1.0 {
+				return "0.1"
+			}
+			return v
+		},
+		DefaultValue: "0.1",
+		Doc:          "the ratio of building interim index rows count with max row count of a flush segment, should set to be < 1.0",
+		Export:       true,
+	}
+	p.InterimIndexBuildRatio.Init(base.mgr)
+
 	p.LoadMemoryUsageFactor = ParamItem{
 		Key:          "queryNode.loadMemoryUsageFactor",
 		Version:      "2.0.0",