diff --git a/configs/milvus.yaml b/configs/milvus.yaml index fdd7078edc..55cebd318e 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -644,8 +644,8 @@ dataCoord: clusteringCompactionUsage: 16 # slot usage of clustering compaction job. mixCompactionUsage: 8 # slot usage of mix compaction job. l0DeleteCompactionUsage: 8 # slot usage of l0 compaction job. - indexTaskSlotUsage: 256 # slot usage of index task - statsTaskSlotUsage: 8 # slot usage of stats task + indexTaskSlotUsage: 64 # slot usage of index task per 512mb + statsTaskSlotUsage: 8 # slot usage of stats task per 512mb analyzeTaskSlotUsage: 65535 # slot usage of analyze task jsonStatsTriggerCount: 10 # jsonkey stats task count per trigger jsonStatsTriggerInterval: 10 # jsonkey task interval per trigger diff --git a/internal/datacoord/util.go b/internal/datacoord/util.go index 251c678945..1b46c78ef3 100644 --- a/internal/datacoord/util.go +++ b/internal/datacoord/util.go @@ -365,25 +365,27 @@ func getSortStatus(sorted bool) string { } func calculateIndexTaskSlot(segmentSize int64) int64 { - if segmentSize > 1000*1024*1024 { - return max(Params.DataCoordCfg.IndexTaskSlotUsage.GetAsInt64(), 1) - } else if segmentSize > 500*1024*1024 { - return max(Params.DataCoordCfg.IndexTaskSlotUsage.GetAsInt64()/4, 1) + defaultSlots := Params.DataCoordCfg.IndexTaskSlotUsage.GetAsInt64() + if segmentSize > 512*1024*1024 { + taskSlot := max(segmentSize/512/1024/1024, 1) * defaultSlots + return max(taskSlot, 1) } else if segmentSize > 100*1024*1024 { - return max(Params.DataCoordCfg.IndexTaskSlotUsage.GetAsInt64()/16, 1) + return max(defaultSlots/4, 1) } else if segmentSize > 10*1024*1024 { - return max(Params.DataCoordCfg.IndexTaskSlotUsage.GetAsInt64()/64, 1) + return max(defaultSlots/16, 1) } - return max(Params.DataCoordCfg.IndexTaskSlotUsage.GetAsInt64()/256, 1) + return max(defaultSlots/64, 1) } func calculateStatsTaskSlot(segmentSize int64) int64 { - if segmentSize > 500*1024*1024 { - return max(Params.DataCoordCfg.StatsTaskSlotUsage.GetAsInt64(), 1) + defaultSlots := Params.DataCoordCfg.StatsTaskSlotUsage.GetAsInt64() + if segmentSize > 512*1024*1024 { + taskSlot := max(segmentSize/512/1024/1024, 1) * defaultSlots + return max(taskSlot, 1) } else if segmentSize > 100*1024*1024 { - return max(Params.DataCoordCfg.StatsTaskSlotUsage.GetAsInt64()/2, 1) + return max(defaultSlots/2, 1) } else if segmentSize > 10*1024*1024 { - return max(Params.DataCoordCfg.StatsTaskSlotUsage.GetAsInt64()/4, 1) + return max(defaultSlots/4, 1) } - return max(Params.DataCoordCfg.StatsTaskSlotUsage.GetAsInt64()/8, 1) + return max(defaultSlots/8, 1) } diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 3680a5c68d..bb89c21ce9 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -4471,8 +4471,8 @@ During compaction, the size of segment # of rows is able to exceed segment max # p.IndexTaskSlotUsage = ParamItem{ Key: "dataCoord.slot.indexTaskSlotUsage", Version: "2.5.8", - Doc: "slot usage of index task", - DefaultValue: "256", + Doc: "slot usage of index task per 512mb", + DefaultValue: "64", PanicIfEmpty: false, Export: true, } @@ -4481,7 +4481,7 @@ During compaction, the size of segment # of rows is able to exceed segment max # p.StatsTaskSlotUsage = ParamItem{ Key: "dataCoord.slot.statsTaskSlotUsage", Version: "2.5.8", - Doc: "slot usage of stats task", + Doc: "slot usage of stats task per 512mb", DefaultValue: "8", PanicIfEmpty: false, Export: true,