From 17a2fd048e787d055cc2fb766955f9b510c60688 Mon Sep 17 00:00:00 2001
From: MrPresent-Han <116052805+MrPresent-Han@users.noreply.github.com>
Date: Thu, 29 Feb 2024 18:15:00 +0800
Subject: [PATCH] feat: support set up knowhere-build-pool-size on
 querynode(#29650) (#30922)

related: #29650

Signed-off-by: MrPresent-Han <chun.han@zilliz.com>
---
 configs/milvus.yaml                    |  1 +
 internal/querynodev2/server.go         |  4 ++++
 pkg/util/paramtable/component_param.go | 22 ++++++++++++++++------
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/configs/milvus.yaml b/configs/milvus.yaml
index df57c561a5..e079cf1095 100644
--- a/configs/milvus.yaml
+++ b/configs/milvus.yaml
@@ -303,6 +303,7 @@ queryNode:
       nlist: 128 # segment index nlist
       nprobe: 16 # nprobe to search segment, based on your accuracy requirement, must smaller than nlist
       memExpansionRate: 1.15 # the ratio of building interim index memory usage to raw data
+      buildParallelRate: 0.5 # the ratio of building interim index parallel matched with cpu num
   loadMemoryUsageFactor: 1 # The multiply factor of calculating the memory usage while loading segments
   enableDisk: false # enable querynode load disk index, and search on disk index
   maxDiskUsagePercentage: 95
diff --git a/internal/querynodev2/server.go b/internal/querynodev2/server.go
index f1f2be6e60..877affce84 100644
--- a/internal/querynodev2/server.go
+++ b/internal/querynodev2/server.go
@@ -215,6 +215,10 @@ func (node *QueryNode) InitSegcore() error {
 	cCPUNum := C.int(hardware.GetCPUNum())
 	C.InitCpuNum(cCPUNum)
 
+	knowhereBuildPoolSize := uint32(float32(paramtable.Get().QueryNodeCfg.InterimIndexBuildParallelRate.GetAsFloat()) * float32(hardware.GetCPUNum()))
+	cKnowhereBuildPoolSize := C.uint32_t(knowhereBuildPoolSize)
+	C.SegcoreSetKnowhereBuildThreadPoolNum(cKnowhereBuildPoolSize)
+
 	cExprBatchSize := C.int64_t(paramtable.Get().QueryNodeCfg.ExprEvalBatchSize.GetAsInt64())
 	C.InitDefaultExprEvalBatchSize(cExprBatchSize)
 
diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go
index 08cdbb956d..ed9c256ec9 100644
--- a/pkg/util/paramtable/component_param.go
+++ b/pkg/util/paramtable/component_param.go
@@ -1917,12 +1917,13 @@ type queryNodeConfig struct {
 	StatsPublishInterval ParamItem `refreshable:"true"`
 
 	// segcore
-	KnowhereThreadPoolSize    ParamItem `refreshable:"false"`
-	ChunkRows                 ParamItem `refreshable:"false"`
-	EnableTempSegmentIndex    ParamItem `refreshable:"false"`
-	InterimIndexNlist         ParamItem `refreshable:"false"`
-	InterimIndexNProbe        ParamItem `refreshable:"false"`
-	InterimIndexMemExpandRate ParamItem `refreshable:"false"`
+	KnowhereThreadPoolSize        ParamItem `refreshable:"false"`
+	ChunkRows                     ParamItem `refreshable:"false"`
+	EnableTempSegmentIndex        ParamItem `refreshable:"false"`
+	InterimIndexNlist             ParamItem `refreshable:"false"`
+	InterimIndexNProbe            ParamItem `refreshable:"false"`
+	InterimIndexMemExpandRate     ParamItem `refreshable:"false"`
+	InterimIndexBuildParallelRate ParamItem `refreshable:"true"`
 
 	// memory limit
 	LoadMemoryUsageFactor               ParamItem `refreshable:"true"`
@@ -2076,6 +2077,15 @@ func (p *queryNodeConfig) init(base *BaseTable) {
 	}
 	p.InterimIndexMemExpandRate.Init(base.mgr)
 
+	p.InterimIndexBuildParallelRate = ParamItem{
+		Key:          "queryNode.segcore.interimIndex.buildParallelRate",
+		Version:      "2.0.0",
+		DefaultValue: "0.5",
+		Doc:          "the ratio of building interim index parallel matched with cpu num",
+		Export:       true,
+	}
+	p.InterimIndexBuildParallelRate.Init(base.mgr)
+
 	p.InterimIndexNProbe = ParamItem{
 		Key:     "queryNode.segcore.interimIndex.nprobe",
 		Version: "2.0.0",