admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: [2.5] As the segment size increases the number of slots should also increase (#40862)

Browse Source 
issue: #40858 
master pr: #39084

Signed-off-by: Cai Zhang <cai.zhang@zilliz.com>
This commit is contained in:
cai.zhang 2025-03-25 14:42:24 +08:00 committed by GitHub
parent 97fc68bf08
commit ec0c25aaec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 19 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
configs/milvus.yaml
View File

@ -644,8 +644,8 @@ dataCoord:
clusteringCompactionUsage: 16 # slot usage of clustering compaction job.
mixCompactionUsage: 8 # slot usage of mix compaction job.
l0DeleteCompactionUsage: 8 # slot usage of l0 compaction job.
indexTaskSlotUsage: 256 # slot usage of index task
statsTaskSlotUsage: 8 # slot usage of stats task
indexTaskSlotUsage: 64 # slot usage of index task per 512mb
statsTaskSlotUsage: 8 # slot usage of stats task per 512mb
analyzeTaskSlotUsage: 65535 # slot usage of analyze task
jsonStatsTriggerCount: 10 # jsonkey stats task count per trigger
jsonStatsTriggerInterval: 10 # jsonkey task interval per trigger

26
internal/datacoord/util.go
View File

@ -365,25 +365,27 @@ func getSortStatus(sorted bool) string {
}
func calculateIndexTaskSlot(segmentSize int64) int64 {
if segmentSize > 1000*1024*1024 {
return max(Params.DataCoordCfg.IndexTaskSlotUsage.GetAsInt64(), 1)
} else if segmentSize > 500*1024*1024 {
return max(Params.DataCoordCfg.IndexTaskSlotUsage.GetAsInt64()/4, 1)
defaultSlots := Params.DataCoordCfg.IndexTaskSlotUsage.GetAsInt64()
if segmentSize > 512*1024*1024 {
taskSlot := max(segmentSize/512/1024/1024, 1) * defaultSlots
return max(taskSlot, 1)
} else if segmentSize > 100*1024*1024 {
return max(Params.DataCoordCfg.IndexTaskSlotUsage.GetAsInt64()/16, 1)
return max(defaultSlots/4, 1)
} else if segmentSize > 10*1024*1024 {
return max(Params.DataCoordCfg.IndexTaskSlotUsage.GetAsInt64()/64, 1)
return max(defaultSlots/16, 1)
}
return max(Params.DataCoordCfg.IndexTaskSlotUsage.GetAsInt64()/256, 1)
return max(defaultSlots/64, 1)
}
func calculateStatsTaskSlot(segmentSize int64) int64 {
if segmentSize > 500*1024*1024 {
return max(Params.DataCoordCfg.StatsTaskSlotUsage.GetAsInt64(), 1)
defaultSlots := Params.DataCoordCfg.StatsTaskSlotUsage.GetAsInt64()
if segmentSize > 512*1024*1024 {
taskSlot := max(segmentSize/512/1024/1024, 1) * defaultSlots
return max(taskSlot, 1)
} else if segmentSize > 100*1024*1024 {
return max(Params.DataCoordCfg.StatsTaskSlotUsage.GetAsInt64()/2, 1)
return max(defaultSlots/2, 1)
} else if segmentSize > 10*1024*1024 {
return max(Params.DataCoordCfg.StatsTaskSlotUsage.GetAsInt64()/4, 1)
return max(defaultSlots/4, 1)
}
return max(Params.DataCoordCfg.StatsTaskSlotUsage.GetAsInt64()/8, 1)
return max(defaultSlots/8, 1)
}

6
pkg/util/paramtable/component_param.go
View File

@ -4471,8 +4471,8 @@ During compaction, the size of segment # of rows is able to exceed segment max #
p.IndexTaskSlotUsage = ParamItem{
Key: "dataCoord.slot.indexTaskSlotUsage",
Version: "2.5.8",
Doc: "slot usage of index task",
DefaultValue: "256",
Doc: "slot usage of index task per 512mb",
DefaultValue: "64",
PanicIfEmpty: false,
Export: true,
}
@ -4481,7 +4481,7 @@ During compaction, the size of segment # of rows is able to exceed segment max #
p.StatsTaskSlotUsage = ParamItem{
Key: "dataCoord.slot.statsTaskSlotUsage",
Version: "2.5.8",
Doc: "slot usage of stats task",
Doc: "slot usage of stats task per 512mb",
DefaultValue: "8",
PanicIfEmpty: false,
Export: true,