admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: Add configs for compaction schedule (#39010)

Browse Source 
Signed-off-by: yangxuan <xuan.yang@zilliz.com>
This commit is contained in:
XuanYang-cn 2025-01-22 16:31:10 +08:00 committed by GitHub
parent 1a1ed07cfa
commit bb8cc6eb85
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 22 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
configs/milvus.yaml
View File

@ -572,6 +572,7 @@ dataCoord:
maxParallelTaskNum: -1 # Deprecated, see datanode.slot.slotCap maxParallelTaskNum: -1 # Deprecated, see datanode.slot.slotCap
dropTolerance: 86400 # Compaction task will be cleaned after finish longer than this time(in seconds) dropTolerance: 86400 # Compaction task will be cleaned after finish longer than this time(in seconds)
gcInterval: 1800 # The time interval in seconds for compaction gc gcInterval: 1800 # The time interval in seconds for compaction gc
scheduleInterval: 500 # The time interval in milliseconds for scheduling compaction tasks. If the configuration setting is below 100ms, it will be ajusted upwards to 100ms
mix: mix:
triggerInterval: 60 # The time interval in seconds to trigger mix compaction triggerInterval: 60 # The time interval in seconds to trigger mix compaction
levelzero: levelzero:

5
internal/datacoord/compaction.go
View File

@ -440,10 +440,11 @@ func (c *compactionPlanHandler) loadMeta() {
} }
func (c *compactionPlanHandler) loopSchedule() { func (c *compactionPlanHandler) loopSchedule() {
log.Info("compactionPlanHandler start loop schedule") interval := paramtable.Get().DataCoordCfg.CompactionScheduleInterval.GetAsDuration(time.Millisecond)
log.Info("compactionPlanHandler start loop schedule", zap.Duration("schedule interval", interval))
defer c.stopWg.Done() defer c.stopWg.Done()
scheduleTicker := time.NewTicker(3 * time.Second) scheduleTicker := time.NewTicker(interval)
defer scheduleTicker.Stop() defer scheduleTicker.Stop()
for { for {
select { select {

19
pkg/util/paramtable/component_param.go
View File

@ -3370,7 +3370,8 @@ type dataCoordConfig struct {
CompactionTimeoutInSeconds ParamItem `refreshable:"true"` CompactionTimeoutInSeconds ParamItem `refreshable:"true"`
CompactionDropToleranceInSeconds ParamItem `refreshable:"true"` CompactionDropToleranceInSeconds ParamItem `refreshable:"true"`
CompactionGCIntervalInSeconds ParamItem `refreshable:"true"` CompactionGCIntervalInSeconds ParamItem `refreshable:"true"`
CompactionCheckIntervalInSeconds ParamItem `refreshable:"false"` CompactionCheckIntervalInSeconds ParamItem `refreshable:"false"` // deprecated
CompactionScheduleInterval ParamItem `refreshable:"false"`
MixCompactionTriggerInterval ParamItem `refreshable:"false"` MixCompactionTriggerInterval ParamItem `refreshable:"false"`
L0CompactionTriggerInterval ParamItem `refreshable:"false"` L0CompactionTriggerInterval ParamItem `refreshable:"false"`
GlobalCompactionInterval ParamItem `refreshable:"false"` GlobalCompactionInterval ParamItem `refreshable:"false"`
@ -3746,6 +3747,22 @@ During compaction, the size of segment # of rows is able to exceed segment max #
} }
p.CompactionCheckIntervalInSeconds.Init(base.mgr) p.CompactionCheckIntervalInSeconds.Init(base.mgr)
p.CompactionScheduleInterval = ParamItem{
Key: "dataCoord.compaction.scheduleInterval",
Version: "2.4.21",
DefaultValue: "500",
Export: true,
Formatter: func(value string) string {
ms := getAsInt64(value)
if ms < 100 {
ms = 100
}
return strconv.FormatInt(ms, 10)
},
Doc: "The time interval in milliseconds for scheduling compaction tasks. If the configuration setting is below 100ms, it will be ajusted upwards to 100ms",
}
p.CompactionScheduleInterval.Init(base.mgr)
p.SingleCompactionRatioThreshold = ParamItem{ p.SingleCompactionRatioThreshold = ParamItem{
Key: "dataCoord.compaction.single.ratio.threshold", Key: "dataCoord.compaction.single.ratio.threshold",
Version: "2.0.0", Version: "2.0.0",