admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Split small segment proportion and small segment compactable proportion (#20996)

Browse Source 
/kind improvement

Signed-off-by: Yuchen Gao <yuchen.gao@zilliz.com>

Signed-off-by: Yuchen Gao <yuchen.gao@zilliz.com>
This commit is contained in:
Ten Thousand Leaves 2022-12-06 14:37:18 +08:00 committed by GitHub
parent 039e9ce4bb
commit 2f5b03fef7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
configs/milvus.yaml
View File

@ -276,6 +276,10 @@ dataCoord:
# `minSizeFromIdleToSealed`, Milvus will automatically seal it. # `minSizeFromIdleToSealed`, Milvus will automatically seal it.
maxIdleTime: 600 # The max idle time of segment in seconds, 10*60. maxIdleTime: 600 # The max idle time of segment in seconds, 10*60.
minSizeFromIdleToSealed: 16 # The min size in MB of segment which can be idle from sealed. minSizeFromIdleToSealed: 16 # The min size in MB of segment which can be idle from sealed.
smallProportion: 0.5 # The segment is considered as "small segment" when its # of rows is smaller than
# (smallProportion * segment max # of rows).
compactableProportion: 0.5 # A compaction will happen on small segments if the segment after compaction will have
# over (compactableProportion * segment max # of rows) rows.
compaction: compaction:
enableAutoCompaction: true enableAutoCompaction: true

2
internal/datacoord/compaction_trigger.go
View File

@ -560,7 +560,7 @@ func (t *compactionTrigger) generatePlans(segments []*SegmentInfo, force bool, c
targetRow += s.GetNumOfRows() targetRow += s.GetNumOfRows()
} }
// only merge if candidate number is large than MinSegmentToMerge or if target row is large enough // only merge if candidate number is large than MinSegmentToMerge or if target row is large enough
if len(bucket) >= Params.DataCoordCfg.MinSegmentToMerge || targetRow > int64(float64(segment.GetMaxRowNum())*Params.DataCoordCfg.SegmentSmallProportion) { if len(bucket) >= Params.DataCoordCfg.MinSegmentToMerge || targetRow > int64(float64(segment.GetMaxRowNum())*Params.DataCoordCfg.SegmentCompactableProportion) {
plan := segmentsToPlan(bucket, compactTime) plan := segmentsToPlan(bucket, compactTime)
log.Info("generate a plan for small candidates", zap.Any("plan", plan), log.Info("generate a plan for small candidates", zap.Any("plan", plan),
zap.Int64("target segment row", targetRow), zap.Int64("target segment size", size)) zap.Int64("target segment row", targetRow), zap.Int64("target segment size", size))

6
internal/util/paramtable/component_param.go
View File

@ -1175,6 +1175,7 @@ type dataCoordConfig struct {
MinSegmentToMerge int MinSegmentToMerge int
MaxSegmentToMerge int MaxSegmentToMerge int
SegmentSmallProportion float64 SegmentSmallProportion float64
SegmentCompactableProportion float64
CompactionTimeoutInSeconds int32 CompactionTimeoutInSeconds int32
CompactionCheckIntervalInSeconds int64 CompactionCheckIntervalInSeconds int64
SingleCompactionRatioThreshold float32 SingleCompactionRatioThreshold float32
@ -1211,6 +1212,7 @@ func (p *dataCoordConfig) init(base *BaseTable) {
p.initCompactionMinSegment() p.initCompactionMinSegment()
p.initCompactionMaxSegment() p.initCompactionMaxSegment()
p.initSegmentSmallProportion() p.initSegmentSmallProportion()
p.initSegmentCompactableProportion()
p.initCompactionTimeoutInSeconds() p.initCompactionTimeoutInSeconds()
p.initCompactionCheckIntervalInSeconds() p.initCompactionCheckIntervalInSeconds()
p.initSingleCompactionRatioThreshold() p.initSingleCompactionRatioThreshold()
@ -1286,6 +1288,10 @@ func (p *dataCoordConfig) initSegmentSmallProportion() {
p.SegmentSmallProportion = p.Base.ParseFloatWithDefault("dataCoord.segment.smallProportion", 0.5) p.SegmentSmallProportion = p.Base.ParseFloatWithDefault("dataCoord.segment.smallProportion", 0.5)
} }
func (p *dataCoordConfig) initSegmentCompactableProportion() {
p.SegmentCompactableProportion = p.Base.ParseFloatWithDefault("dataCoord.segment.compactableProportion", 0.5)
}
// compaction execution timeout // compaction execution timeout
func (p *dataCoordConfig) initCompactionTimeoutInSeconds() { func (p *dataCoordConfig) initCompactionTimeoutInSeconds() {
p.CompactionTimeoutInSeconds = p.Base.ParseInt32WithDefault("dataCoord.compaction.timeout", 60*3) p.CompactionTimeoutInSeconds = p.Base.ParseInt32WithDefault("dataCoord.compaction.timeout", 60*3)